ilovetools 0.2.16__tar.gz → 0.2.17__tar.gz

{ilovetools-0.2.16/ilovetools.egg-info → ilovetools-0.2.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ilovetools
-Version: 0.2.16
+Version: 0.2.17
 Summary: A comprehensive Python utility library with modular tools for AI/ML, data processing, and daily programming needs
 Home-page: https://github.com/AliMehdi512/ilovetools
 Author: Ali Mehdi
@@ -11,7 +11,7 @@ Project-URL: Repository, https://github.com/AliMehdi512/ilovetools
 Project-URL: Issues, https://github.com/AliMehdi512/ilovetools/issues
 Project-URL: Bug Reports, https://github.com/AliMehdi512/ilovetools/issues
 Project-URL: Source, https://github.com/AliMehdi512/ilovetools
-Keywords: utilities,tools,ai,ml,data-processing,automation,gradient-descent,optimization,regularization,loss-functions,activation-functions,optimizers,adam,adamw,radam,lamb
+Keywords: utilities,tools,ai,ml,data-processing,automation,batch-normalization,layer-normalization,group-normalization,instance-normalization,weight-normalization,deep-learning,transformers
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: Topic :: Software Development :: Libraries :: Python Modules

{ilovetools-0.2.16 → ilovetools-0.2.17}/ilovetools/__init__.py

@@ -2,8 +2,8 @@
 ilovetools - A comprehensive Python utility library
 """
 
-__version__ = "0.2.15"
-# release marker: 0.2.15
+__version__ = "0.2.16"
+# release marker: 0.2.16
 __author__ = "Ali Mehdi"
 __email__ = "ali.mehdi.dev579@gmail.com"
 

ilovetools-0.2.17/ilovetools/ml/normalization.py

@@ -0,0 +1,523 @@
+"""
+Normalization Techniques for Neural Networks
+
+This module provides various normalization techniques used in deep learning:
+- Batch Normalization
+- Layer Normalization
+- Group Normalization
+- Instance Normalization
+- Weight Normalization
+
+All normalization functions support both training and inference modes.
+"""
+
+import numpy as np
+from typing import Tuple, Optional
+
+
+# ============================================================================
+# BATCH NORMALIZATION
+# ============================================================================
+
+def batch_normalization(
+    x: np.ndarray,
+    gamma: np.ndarray,
+    beta: np.ndarray,
+    running_mean: Optional[np.ndarray] = None,
+    running_var: Optional[np.ndarray] = None,
+    training: bool = True,
+    momentum: float = 0.9,
+    epsilon: float = 1e-5
+) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
+    """
+    Batch Normalization
+    
+    Normalizes activations across the batch dimension. Reduces internal covariate shift
+    and allows higher learning rates.
+    
+    Formula: y = gamma * (x - mean) / sqrt(var + epsilon) + beta
+    
+    Args:
+        x: Input tensor of shape (N, D) or (N, C, H, W)
+        gamma: Scale parameter of shape (D,) or (C,)
+        beta: Shift parameter of shape (D,) or (C,)
+        running_mean: Running mean for inference (updated during training)
+        running_var: Running variance for inference (updated during training)
+        training: Whether in training mode
+        momentum: Momentum for running statistics (default: 0.9)
+        epsilon: Small constant for numerical stability (default: 1e-5)
+        
+    Returns:
+        Tuple of (normalized_output, updated_running_mean, updated_running_var)
+        
+    Example:
+        >>> x = np.random.randn(32, 64)  # Batch of 32, 64 features
+        >>> gamma = np.ones(64)
+        >>> beta = np.zeros(64)
+        >>> out, mean, var = batch_normalization(x, gamma, beta, training=True)
+        >>> print(out.shape)  # (32, 64)
+    """
+    if running_mean is None:
+        running_mean = np.zeros(gamma.shape)
+    if running_var is None:
+        running_var = np.ones(gamma.shape)
+    
+    if training:
+        # Compute batch statistics
+        if x.ndim == 2:
+            # Fully connected layer: (N, D)
+            batch_mean = np.mean(x, axis=0)
+            batch_var = np.var(x, axis=0)
+        elif x.ndim == 4:
+            # Convolutional layer: (N, C, H, W)
+            batch_mean = np.mean(x, axis=(0, 2, 3), keepdims=True)
+            batch_var = np.var(x, axis=(0, 2, 3), keepdims=True)
+        else:
+            raise ValueError(f"Unsupported input shape: {x.shape}")
+        
+        # Normalize
+        x_normalized = (x - batch_mean) / np.sqrt(batch_var + epsilon)
+        
+        # Update running statistics
+        running_mean = momentum * running_mean + (1 - momentum) * batch_mean.squeeze()
+        running_var = momentum * running_var + (1 - momentum) * batch_var.squeeze()
+    else:
+        # Use running statistics for inference
+        if x.ndim == 2:
+            x_normalized = (x - running_mean) / np.sqrt(running_var + epsilon)
+        elif x.ndim == 4:
+            mean_reshaped = running_mean.reshape(1, -1, 1, 1)
+            var_reshaped = running_var.reshape(1, -1, 1, 1)
+            x_normalized = (x - mean_reshaped) / np.sqrt(var_reshaped + epsilon)
+    
+    # Scale and shift
+    if x.ndim == 2:
+        out = gamma * x_normalized + beta
+    elif x.ndim == 4:
+        gamma_reshaped = gamma.reshape(1, -1, 1, 1)
+        beta_reshaped = beta.reshape(1, -1, 1, 1)
+        out = gamma_reshaped * x_normalized + beta_reshaped
+    
+    return out, running_mean, running_var
+
+
+def batch_norm_forward(
+    x: np.ndarray,
+    gamma: np.ndarray,
+    beta: np.ndarray,
+    bn_params: dict
+) -> Tuple[np.ndarray, dict]:
+    """
+    Forward pass for batch normalization with cache
+    
+    Args:
+        x: Input data
+        gamma: Scale parameter
+        beta: Shift parameter
+        bn_params: Dictionary with keys:
+            - mode: 'train' or 'test'
+            - eps: Epsilon for numerical stability
+            - momentum: Momentum for running statistics
+            - running_mean: Running mean
+            - running_var: Running variance
+            
+    Returns:
+        Tuple of (output, cache)
+        
+    Example:
+        >>> x = np.random.randn(32, 64)
+        >>> gamma = np.ones(64)
+        >>> beta = np.zeros(64)
+        >>> bn_params = {
+        ...     'mode': 'train',
+        ...     'eps': 1e-5,
+        ...     'momentum': 0.9,
+        ...     'running_mean': np.zeros(64),
+        ...     'running_var': np.ones(64)
+        ... }
+        >>> out, cache = batch_norm_forward(x, gamma, beta, bn_params)
+    """
+    mode = bn_params.get('mode', 'train')
+    eps = bn_params.get('eps', 1e-5)
+    momentum = bn_params.get('momentum', 0.9)
+    
+    running_mean = bn_params.get('running_mean', np.zeros(gamma.shape))
+    running_var = bn_params.get('running_var', np.ones(gamma.shape))
+    
+    training = (mode == 'train')
+    
+    out, running_mean, running_var = batch_normalization(
+        x, gamma, beta, running_mean, running_var,
+        training=training, momentum=momentum, epsilon=eps
+    )
+    
+    # Update running statistics in params
+    bn_params['running_mean'] = running_mean
+    bn_params['running_var'] = running_var
+    
+    # Cache for backward pass
+    cache = {
+        'x': x,
+        'gamma': gamma,
+        'beta': beta,
+        'eps': eps,
+        'x_normalized': (x - running_mean) / np.sqrt(running_var + eps),
+        'mean': running_mean,
+        'var': running_var
+    }
+    
+    return out, cache
+
+
+# ============================================================================
+# LAYER NORMALIZATION
+# ============================================================================
+
+def layer_normalization(
+    x: np.ndarray,
+    gamma: np.ndarray,
+    beta: np.ndarray,
+    epsilon: float = 1e-5
+) -> np.ndarray:
+    """
+    Layer Normalization
+    
+    Normalizes activations across the feature dimension. Independent of batch size,
+    making it suitable for RNNs and Transformers.
+    
+    Formula: y = gamma * (x - mean) / sqrt(var + epsilon) + beta
+    
+    Args:
+        x: Input tensor of shape (N, D) or (N, L, D)
+        gamma: Scale parameter of shape (D,)
+        beta: Shift parameter of shape (D,)
+        epsilon: Small constant for numerical stability (default: 1e-5)
+        
+    Returns:
+        Normalized output of same shape as input
+        
+    Example:
+        >>> x = np.random.randn(32, 512)  # Batch of 32, 512 features
+        >>> gamma = np.ones(512)
+        >>> beta = np.zeros(512)
+        >>> out = layer_normalization(x, gamma, beta)
+        >>> print(out.shape)  # (32, 512)
+        
+        >>> # For sequences (Transformers)
+        >>> x = np.random.randn(32, 10, 512)  # Batch 32, seq len 10, dim 512
+        >>> out = layer_normalization(x, gamma, beta)
+        >>> print(out.shape)  # (32, 10, 512)
+    """
+    # Compute mean and variance across feature dimension
+    if x.ndim == 2:
+        # (N, D) - Fully connected
+        mean = np.mean(x, axis=1, keepdims=True)
+        var = np.var(x, axis=1, keepdims=True)
+    elif x.ndim == 3:
+        # (N, L, D) - Sequence data (Transformers)
+        mean = np.mean(x, axis=2, keepdims=True)
+        var = np.var(x, axis=2, keepdims=True)
+    else:
+        raise ValueError(f"Unsupported input shape: {x.shape}")
+    
+    # Normalize
+    x_normalized = (x - mean) / np.sqrt(var + epsilon)
+    
+    # Scale and shift
+    out = gamma * x_normalized + beta
+    
+    return out
+
+
+def layer_norm_forward(
+    x: np.ndarray,
+    gamma: np.ndarray,
+    beta: np.ndarray,
+    epsilon: float = 1e-5
+) -> Tuple[np.ndarray, dict]:
+    """
+    Forward pass for layer normalization with cache
+    
+    Args:
+        x: Input data
+        gamma: Scale parameter
+        beta: Shift parameter
+        epsilon: Epsilon for numerical stability
+        
+    Returns:
+        Tuple of (output, cache)
+        
+    Example:
+        >>> x = np.random.randn(32, 512)
+        >>> gamma = np.ones(512)
+        >>> beta = np.zeros(512)
+        >>> out, cache = layer_norm_forward(x, gamma, beta)
+    """
+    # Compute statistics
+    if x.ndim == 2:
+        mean = np.mean(x, axis=1, keepdims=True)
+        var = np.var(x, axis=1, keepdims=True)
+    elif x.ndim == 3:
+        mean = np.mean(x, axis=2, keepdims=True)
+        var = np.var(x, axis=2, keepdims=True)
+    else:
+        raise ValueError(f"Unsupported input shape: {x.shape}")
+    
+    # Normalize
+    x_normalized = (x - mean) / np.sqrt(var + epsilon)
+    
+    # Scale and shift
+    out = gamma * x_normalized + beta
+    
+    # Cache for backward pass
+    cache = {
+        'x': x,
+        'gamma': gamma,
+        'beta': beta,
+        'eps': epsilon,
+        'x_normalized': x_normalized,
+        'mean': mean,
+        'var': var
+    }
+    
+    return out, cache
+
+
+# ============================================================================
+# GROUP NORMALIZATION
+# ============================================================================
+
+def group_normalization(
+    x: np.ndarray,
+    gamma: np.ndarray,
+    beta: np.ndarray,
+    num_groups: int = 32,
+    epsilon: float = 1e-5
+) -> np.ndarray:
+    """
+    Group Normalization
+    
+    Divides channels into groups and normalizes within each group.
+    Works well with small batch sizes.
+    
+    Args:
+        x: Input tensor of shape (N, C, H, W)
+        gamma: Scale parameter of shape (C,)
+        beta: Shift parameter of shape (C,)
+        num_groups: Number of groups (default: 32)
+        epsilon: Small constant for numerical stability
+        
+    Returns:
+        Normalized output of same shape as input
+        
+    Example:
+        >>> x = np.random.randn(8, 64, 32, 32)  # Small batch
+        >>> gamma = np.ones(64)
+        >>> beta = np.zeros(64)
+        >>> out = group_normalization(x, gamma, beta, num_groups=32)
+        >>> print(out.shape)  # (8, 64, 32, 32)
+    """
+    N, C, H, W = x.shape
+    
+    if C % num_groups != 0:
+        raise ValueError(f"Number of channels ({C}) must be divisible by num_groups ({num_groups})")
+    
+    # Reshape to (N, num_groups, C // num_groups, H, W)
+    x_grouped = x.reshape(N, num_groups, C // num_groups, H, W)
+    
+    # Compute mean and variance per group
+    mean = np.mean(x_grouped, axis=(2, 3, 4), keepdims=True)
+    var = np.var(x_grouped, axis=(2, 3, 4), keepdims=True)
+    
+    # Normalize
+    x_normalized = (x_grouped - mean) / np.sqrt(var + epsilon)
+    
+    # Reshape back
+    x_normalized = x_normalized.reshape(N, C, H, W)
+    
+    # Scale and shift
+    gamma_reshaped = gamma.reshape(1, -1, 1, 1)
+    beta_reshaped = beta.reshape(1, -1, 1, 1)
+    out = gamma_reshaped * x_normalized + beta_reshaped
+    
+    return out
+
+
+# ============================================================================
+# INSTANCE NORMALIZATION
+# ============================================================================
+
+def instance_normalization(
+    x: np.ndarray,
+    gamma: np.ndarray,
+    beta: np.ndarray,
+    epsilon: float = 1e-5
+) -> np.ndarray:
+    """
+    Instance Normalization
+    
+    Normalizes each sample independently. Used in style transfer and GANs.
+    
+    Args:
+        x: Input tensor of shape (N, C, H, W)
+        gamma: Scale parameter of shape (C,)
+        beta: Shift parameter of shape (C,)
+        epsilon: Small constant for numerical stability
+        
+    Returns:
+        Normalized output of same shape as input
+        
+    Example:
+        >>> x = np.random.randn(8, 64, 32, 32)
+        >>> gamma = np.ones(64)
+        >>> beta = np.zeros(64)
+        >>> out = instance_normalization(x, gamma, beta)
+        >>> print(out.shape)  # (8, 64, 32, 32)
+    """
+    N, C, H, W = x.shape
+    
+    # Compute mean and variance per instance per channel
+    mean = np.mean(x, axis=(2, 3), keepdims=True)
+    var = np.var(x, axis=(2, 3), keepdims=True)
+    
+    # Normalize
+    x_normalized = (x - mean) / np.sqrt(var + epsilon)
+    
+    # Scale and shift
+    gamma_reshaped = gamma.reshape(1, -1, 1, 1)
+    beta_reshaped = beta.reshape(1, -1, 1, 1)
+    out = gamma_reshaped * x_normalized + beta_reshaped
+    
+    return out
+
+
+# ============================================================================
+# WEIGHT NORMALIZATION
+# ============================================================================
+
+def weight_normalization(
+    w: np.ndarray,
+    g: Optional[np.ndarray] = None,
+    axis: int = 0
+) -> Tuple[np.ndarray, np.ndarray]:
+    """
+    Weight Normalization
+    
+    Decouples the magnitude and direction of weight vectors.
+    Faster than batch normalization.
+    
+    Formula: w = g * (v / ||v||)
+    
+    Args:
+        w: Weight matrix
+        g: Magnitude parameter (if None, computed from w)
+        axis: Axis along which to normalize (default: 0)
+        
+    Returns:
+        Tuple of (normalized_weights, magnitude)
+        
+    Example:
+        >>> w = np.random.randn(512, 256)  # Weight matrix
+        >>> w_norm, g = weight_normalization(w)
+        >>> print(w_norm.shape)  # (512, 256)
+        >>> print(g.shape)  # (512,) or (256,) depending on axis
+    """
+    # Compute norm along specified axis
+    norm = np.linalg.norm(w, axis=axis, keepdims=True)
+    
+    # Normalize direction
+    v = w / (norm + 1e-8)
+    
+    # Compute or use provided magnitude
+    if g is None:
+        g = norm.squeeze()
+    
+    # Reconstruct weights
+    if axis == 0:
+        w_normalized = g.reshape(-1, 1) * v
+    else:
+        w_normalized = g.reshape(1, -1) * v
+    
+    return w_normalized, g
+
+
+# ============================================================================
+# UTILITY FUNCTIONS
+# ============================================================================
+
+def create_normalization_params(
+    num_features: int,
+    norm_type: str = 'batch'
+) -> dict:
+    """
+    Create parameters for normalization layers
+    
+    Args:
+        num_features: Number of features/channels
+        norm_type: Type of normalization ('batch', 'layer', 'group', 'instance')
+        
+    Returns:
+        Dictionary with initialized parameters
+        
+    Example:
+        >>> params = create_normalization_params(64, 'batch')
+        >>> print(params.keys())
+        dict_keys(['gamma', 'beta', 'running_mean', 'running_var'])
+    """
+    params = {
+        'gamma': np.ones(num_features),
+        'beta': np.zeros(num_features)
+    }
+    
+    if norm_type == 'batch':
+        params['running_mean'] = np.zeros(num_features)
+        params['running_var'] = np.ones(num_features)
+    
+    return params
+
+
+def apply_normalization(
+    x: np.ndarray,
+    norm_type: str,
+    gamma: np.ndarray,
+    beta: np.ndarray,
+    **kwargs
+) -> np.ndarray:
+    """
+    Apply normalization by type name
+    
+    Args:
+        x: Input tensor
+        norm_type: Type of normalization
+        gamma: Scale parameter
+        beta: Shift parameter
+        **kwargs: Additional arguments for specific normalization
+        
+    Returns:
+        Normalized output
+        
+    Example:
+        >>> x = np.random.randn(32, 64)
+        >>> gamma = np.ones(64)
+        >>> beta = np.zeros(64)
+        >>> out = apply_normalization(x, 'layer', gamma, beta)
+    """
+    if norm_type == 'batch':
+        out, _, _ = batch_normalization(x, gamma, beta, **kwargs)
+        return out
+    elif norm_type == 'layer':
+        return layer_normalization(x, gamma, beta, **kwargs)
+    elif norm_type == 'group':
+        return group_normalization(x, gamma, beta, **kwargs)
+    elif norm_type == 'instance':
+        return instance_normalization(x, gamma, beta, **kwargs)
+    else:
+        raise ValueError(f"Unknown normalization type: {norm_type}")
+
+
+# Aliases for convenience
+batchnorm = batch_normalization
+layernorm = layer_normalization
+groupnorm = group_normalization
+instancenorm = instance_normalization
+weightnorm = weight_normalization

{ilovetools-0.2.16 → ilovetools-0.2.17/ilovetools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ilovetools
-Version: 0.2.16
+Version: 0.2.17
 Summary: A comprehensive Python utility library with modular tools for AI/ML, data processing, and daily programming needs
 Home-page: https://github.com/AliMehdi512/ilovetools
 Author: Ali Mehdi
@@ -11,7 +11,7 @@ Project-URL: Repository, https://github.com/AliMehdi512/ilovetools
 Project-URL: Issues, https://github.com/AliMehdi512/ilovetools/issues
 Project-URL: Bug Reports, https://github.com/AliMehdi512/ilovetools/issues
 Project-URL: Source, https://github.com/AliMehdi512/ilovetools
-Keywords: utilities,tools,ai,ml,data-processing,automation,gradient-descent,optimization,regularization,loss-functions,activation-functions,optimizers,adam,adamw,radam,lamb
+Keywords: utilities,tools,ai,ml,data-processing,automation,batch-normalization,layer-normalization,group-normalization,instance-normalization,weight-normalization,deep-learning,transformers
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: Topic :: Software Development :: Libraries :: Python Modules

{ilovetools-0.2.16 → ilovetools-0.2.17}/ilovetools.egg-info/SOURCES.txt

@@ -43,6 +43,7 @@ ilovetools/ml/interpretation.py
 ilovetools/ml/loss_functions.py
 ilovetools/ml/metrics.py
 ilovetools/ml/neural_network.py
+ilovetools/ml/normalization.py
 ilovetools/ml/optimizers.py
 ilovetools/ml/pipeline.py
 ilovetools/ml/regularization.py
@@ -66,5 +67,6 @@ tests/test_activations.py
 tests/test_gradient_descent.py
 tests/test_loss_functions.py
 tests/test_neural_network.py
+tests/test_normalization.py
 tests/test_optimizers.py
 tests/test_regularization.py

{ilovetools-0.2.16 → ilovetools-0.2.17}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ilovetools"
-version = "0.2.16"
+version = "0.2.17"
 description = "A comprehensive Python utility library with modular tools for AI/ML, data processing, and daily programming needs"
 readme = "README.md"
 requires-python = ">=3.8"
@@ -12,7 +12,7 @@ license = "MIT"
 authors = [
     {name = "Ali Mehdi", email = "ali.mehdi.dev579@gmail.com"}
 ]
-keywords = ["utilities", "tools", "ai", "ml", "data-processing", "automation", "gradient-descent", "optimization", "regularization", "loss-functions", "activation-functions", "optimizers", "adam", "adamw", "radam", "lamb"]
+keywords = ["utilities", "tools", "ai", "ml", "data-processing", "automation", "batch-normalization", "layer-normalization", "group-normalization", "instance-normalization", "weight-normalization", "deep-learning", "transformers"]
 classifiers = [
     "Development Status :: 3 - Alpha",
     "Intended Audience :: Developers",

{ilovetools-0.2.16 → ilovetools-0.2.17}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="ilovetools",
-    version="0.2.16",
+    version="0.2.17",
     author="Ali Mehdi",
     author_email="ali.mehdi.dev579@gmail.com",
     description="A comprehensive Python utility library with modular tools for AI/ML, data processing, and daily programming needs",
@@ -57,7 +57,7 @@ setup(
             "soundfile>=0.12.0",
         ],
     },
-    keywords="utilities, tools, ai, ml, data-processing, automation, python-library, neural-networks, gradient-descent, optimization, regularization, loss-functions, activation-functions, optimizers, adam, adamw, radam, lamb",
+    keywords="utilities, tools, ai, ml, data-processing, automation, python-library, neural-networks, batch-normalization, layer-normalization, group-normalization, instance-normalization, weight-normalization, deep-learning, transformers",
     project_urls={
         "Bug Reports": "https://github.com/AliMehdi512/ilovetools/issues",
         "Source": "https://github.com/AliMehdi512/ilovetools",

ilovetools-0.2.17/tests/test_normalization.py

@@ -0,0 +1,439 @@
+"""
+Tests for normalization techniques module
+"""
+
+import numpy as np
+import sys
+import os
+
+# Add parent directory to path
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+
+from ilovetools.ml.normalization import (
+    # Main functions
+    batch_normalization,
+    layer_normalization,
+    group_normalization,
+    instance_normalization,
+    weight_normalization,
+    # Forward passes with cache
+    batch_norm_forward,
+    layer_norm_forward,
+    # Utilities
+    create_normalization_params,
+    apply_normalization,
+    # Aliases
+    batchnorm,
+    layernorm,
+    groupnorm,
+    instancenorm,
+    weightnorm,
+)
+
+
+def test_batch_normalization():
+    """Test Batch Normalization"""
+    print("Testing batch_normalization...")
+    
+    # Test 2D input (fully connected)
+    x = np.random.randn(32, 64)
+    gamma = np.ones(64)
+    beta = np.zeros(64)
+    
+    out, running_mean, running_var = batch_normalization(
+        x, gamma, beta, training=True
+    )
+    
+    assert out.shape == x.shape, "Output shape should match input"
+    assert running_mean.shape == (64,), "Running mean shape incorrect"
+    assert running_var.shape == (64,), "Running var shape incorrect"
+    
+    # Check normalization (mean ≈ 0, var ≈ 1)
+    assert np.abs(np.mean(out)) < 0.1, "Mean should be close to 0"
+    assert np.abs(np.var(out) - 1.0) < 0.2, "Variance should be close to 1"
+    
+    print("✓ batch_normalization passed")
+
+
+def test_batch_normalization_4d():
+    """Test Batch Normalization with 4D input (CNN)"""
+    print("Testing batch_normalization with 4D input...")
+    
+    # Test 4D input (convolutional)
+    x = np.random.randn(8, 32, 16, 16)  # (N, C, H, W)
+    gamma = np.ones(32)
+    beta = np.zeros(32)
+    
+    out, running_mean, running_var = batch_normalization(
+        x, gamma, beta, training=True
+    )
+    
+    assert out.shape == x.shape, "Output shape should match input"
+    assert running_mean.shape == (32,), "Running mean shape incorrect"
+    
+    print("✓ batch_normalization 4D passed")
+
+
+def test_batch_normalization_inference():
+    """Test Batch Normalization in inference mode"""
+    print("Testing batch_normalization inference mode...")
+    
+    x = np.random.randn(32, 64)
+    gamma = np.ones(64)
+    beta = np.zeros(64)
+    running_mean = np.random.randn(64)
+    running_var = np.abs(np.random.randn(64))
+    
+    out, _, _ = batch_normalization(
+        x, gamma, beta, running_mean, running_var, training=False
+    )
+    
+    assert out.shape == x.shape, "Output shape should match input"
+    print("✓ batch_normalization inference passed")
+
+
+def test_layer_normalization():
+    """Test Layer Normalization"""
+    print("Testing layer_normalization...")
+    
+    # Test 2D input
+    x = np.random.randn(32, 512)
+    gamma = np.ones(512)
+    beta = np.zeros(512)
+    
+    out = layer_normalization(x, gamma, beta)
+    
+    assert out.shape == x.shape, "Output shape should match input"
+    
+    # Check normalization per sample
+    for i in range(32):
+        sample_mean = np.mean(out[i])
+        sample_var = np.var(out[i])
+        assert np.abs(sample_mean) < 0.1, f"Sample {i} mean should be close to 0"
+        assert np.abs(sample_var - 1.0) < 0.2, f"Sample {i} variance should be close to 1"
+    
+    print("✓ layer_normalization passed")
+
+
+def test_layer_normalization_3d():
+    """Test Layer Normalization with 3D input (sequences)"""
+    print("Testing layer_normalization with 3D input...")
+    
+    # Test 3D input (sequences for Transformers)
+    x = np.random.randn(32, 10, 512)  # (N, L, D)
+    gamma = np.ones(512)
+    beta = np.zeros(512)
+    
+    out = layer_normalization(x, gamma, beta)
+    
+    assert out.shape == x.shape, "Output shape should match input"
+    
+    # Check normalization per sample per timestep
+    for i in range(32):
+        for t in range(10):
+            token_mean = np.mean(out[i, t])
+            token_var = np.var(out[i, t])
+            assert np.abs(token_mean) < 0.1, "Token mean should be close to 0"
+            assert np.abs(token_var - 1.0) < 0.2, "Token variance should be close to 1"
+    
+    print("✓ layer_normalization 3D passed")
+
+
+def test_group_normalization():
+    """Test Group Normalization"""
+    print("Testing group_normalization...")
+    
+    x = np.random.randn(8, 64, 32, 32)  # (N, C, H, W)
+    gamma = np.ones(64)
+    beta = np.zeros(64)
+    
+    out = group_normalization(x, gamma, beta, num_groups=32)
+    
+    assert out.shape == x.shape, "Output shape should match input"
+    print("✓ group_normalization passed")
+
+
+def test_group_normalization_invalid_groups():
+    """Test Group Normalization with invalid number of groups"""
+    print("Testing group_normalization with invalid groups...")
+    
+    x = np.random.randn(8, 64, 32, 32)
+    gamma = np.ones(64)
+    beta = np.zeros(64)
+    
+    try:
+        out = group_normalization(x, gamma, beta, num_groups=30)
+        assert False, "Should raise ValueError"
+    except ValueError as e:
+        assert "divisible" in str(e).lower()
+    
+    print("✓ group_normalization invalid groups passed")
+
+
+def test_instance_normalization():
+    """Test Instance Normalization"""
+    print("Testing instance_normalization...")
+    
+    x = np.random.randn(8, 64, 32, 32)  # (N, C, H, W)
+    gamma = np.ones(64)
+    beta = np.zeros(64)
+    
+    out = instance_normalization(x, gamma, beta)
+    
+    assert out.shape == x.shape, "Output shape should match input"
+    
+    # Check normalization per instance per channel
+    for n in range(8):
+        for c in range(64):
+            channel_mean = np.mean(out[n, c])
+            channel_var = np.var(out[n, c])
+            assert np.abs(channel_mean) < 0.1, "Channel mean should be close to 0"
+            assert np.abs(channel_var - 1.0) < 0.2, "Channel variance should be close to 1"
+    
+    print("✓ instance_normalization passed")
+
+
+def test_weight_normalization():
+    """Test Weight Normalization"""
+    print("Testing weight_normalization...")
+    
+    w = np.random.randn(512, 256)
+    
+    w_norm, g = weight_normalization(w, axis=0)
+    
+    assert w_norm.shape == w.shape, "Output shape should match input"
+    assert g.shape == (512,), "Magnitude shape incorrect"
+    
+    # Check that each row has unit norm
+    for i in range(512):
+        row_norm = np.linalg.norm(w_norm[i])
+        expected_norm = g[i]
+        assert np.abs(row_norm - expected_norm) < 1e-5, "Row norm should match magnitude"
+    
+    print("✓ weight_normalization passed")
+
+
+def test_weight_normalization_axis1():
+    """Test Weight Normalization with axis=1"""
+    print("Testing weight_normalization with axis=1...")
+    
+    w = np.random.randn(512, 256)
+    
+    w_norm, g = weight_normalization(w, axis=1)
+    
+    assert w_norm.shape == w.shape, "Output shape should match input"
+    assert g.shape == (256,), "Magnitude shape incorrect"
+    
+    print("✓ weight_normalization axis=1 passed")
+
+
+def test_batch_norm_forward():
+    """Test Batch Normalization forward with cache"""
+    print("Testing batch_norm_forward...")
+    
+    x = np.random.randn(32, 64)
+    gamma = np.ones(64)
+    beta = np.zeros(64)
+    
+    bn_params = {
+        'mode': 'train',
+        'eps': 1e-5,
+        'momentum': 0.9,
+        'running_mean': np.zeros(64),
+        'running_var': np.ones(64)
+    }
+    
+    out, cache = batch_norm_forward(x, gamma, beta, bn_params)
+    
+    assert out.shape == x.shape, "Output shape should match input"
+    assert 'x' in cache, "Cache should contain input"
+    assert 'gamma' in cache, "Cache should contain gamma"
+    assert 'x_normalized' in cache, "Cache should contain normalized input"
+    
+    print("✓ batch_norm_forward passed")
+
+
+def test_layer_norm_forward():
+    """Test Layer Normalization forward with cache"""
+    print("Testing layer_norm_forward...")
+    
+    x = np.random.randn(32, 512)
+    gamma = np.ones(512)
+    beta = np.zeros(512)
+    
+    out, cache = layer_norm_forward(x, gamma, beta)
+    
+    assert out.shape == x.shape, "Output shape should match input"
+    assert 'x' in cache, "Cache should contain input"
+    assert 'gamma' in cache, "Cache should contain gamma"
+    assert 'x_normalized' in cache, "Cache should contain normalized input"
+    
+    print("✓ layer_norm_forward passed")
+
+
+def test_create_normalization_params():
+    """Test create_normalization_params utility"""
+    print("Testing create_normalization_params...")
+    
+    # Test batch norm params
+    params = create_normalization_params(64, 'batch')
+    assert 'gamma' in params, "Should have gamma"
+    assert 'beta' in params, "Should have beta"
+    assert 'running_mean' in params, "Batch norm should have running_mean"
+    assert 'running_var' in params, "Batch norm should have running_var"
+    assert params['gamma'].shape == (64,), "Gamma shape incorrect"
+    
+    # Test layer norm params
+    params = create_normalization_params(512, 'layer')
+    assert 'gamma' in params, "Should have gamma"
+    assert 'beta' in params, "Should have beta"
+    assert 'running_mean' not in params, "Layer norm should not have running_mean"
+    
+    print("✓ create_normalization_params passed")
+
+
+def test_apply_normalization():
+    """Test apply_normalization utility"""
+    print("Testing apply_normalization...")
+    
+    x = np.random.randn(32, 64)
+    gamma = np.ones(64)
+    beta = np.zeros(64)
+    
+    # Test batch norm
+    out = apply_normalization(x, 'batch', gamma, beta, training=True)
+    assert out.shape == x.shape, "Batch norm output shape incorrect"
+    
+    # Test layer norm
+    out = apply_normalization(x, 'layer', gamma, beta)
+    assert out.shape == x.shape, "Layer norm output shape incorrect"
+    
+    print("✓ apply_normalization passed")
+
+
+def test_apply_normalization_invalid():
+    """Test apply_normalization with invalid type"""
+    print("Testing apply_normalization with invalid type...")
+    
+    x = np.random.randn(32, 64)
+    gamma = np.ones(64)
+    beta = np.zeros(64)
+    
+    try:
+        out = apply_normalization(x, 'invalid', gamma, beta)
+        assert False, "Should raise ValueError"
+    except ValueError as e:
+        assert "unknown" in str(e).lower()
+    
+    print("✓ apply_normalization invalid type passed")
+
+
+def test_aliases():
+    """Test function aliases"""
+    print("Testing aliases...")
+    
+    x = np.random.randn(32, 64)
+    gamma = np.ones(64)
+    beta = np.zeros(64)
+    
+    # Test batchnorm alias
+    out1, _, _ = batchnorm(x, gamma, beta, training=True)
+    out2, _, _ = batch_normalization(x, gamma, beta, training=True)
+    assert np.allclose(out1, out2), "Batchnorm alias should work"
+    
+    # Test layernorm alias
+    out1 = layernorm(x, gamma, beta)
+    out2 = layer_normalization(x, gamma, beta)
+    assert np.allclose(out1, out2), "Layernorm alias should work"
+    
+    print("✓ aliases passed")
+
+
+def test_batch_vs_layer_norm():
+    """Test difference between batch and layer normalization"""
+    print("Testing batch vs layer normalization...")
+    
+    x = np.random.randn(32, 64)
+    gamma = np.ones(64)
+    beta = np.zeros(64)
+    
+    # Batch norm normalizes across batch
+    out_batch, _, _ = batch_normalization(x, gamma, beta, training=True)
+    
+    # Layer norm normalizes across features
+    out_layer = layer_normalization(x, gamma, beta)
+    
+    # They should produce different results
+    assert not np.allclose(out_batch, out_layer), "Batch and layer norm should differ"
+    
+    print("✓ batch vs layer norm comparison passed")
+
+
+def test_normalization_stability():
+    """Test numerical stability with extreme values"""
+    print("Testing normalization stability...")
+    
+    # Test with very large values
+    x = np.random.randn(32, 64) * 1000
+    gamma = np.ones(64)
+    beta = np.zeros(64)
+    
+    out, _, _ = batch_normalization(x, gamma, beta, training=True)
+    assert not np.any(np.isnan(out)), "Should not produce NaN"
+    assert not np.any(np.isinf(out)), "Should not produce Inf"
+    
+    # Test with very small values
+    x = np.random.randn(32, 64) * 1e-10
+    out, _, _ = batch_normalization(x, gamma, beta, training=True)
+    assert not np.any(np.isnan(out)), "Should not produce NaN with small values"
+    
+    print("✓ normalization stability passed")
+
+
+def run_all_tests():
+    """Run all tests"""
+    print("\n" + "="*60)
+    print("NORMALIZATION TECHNIQUES MODULE TESTS")
+    print("="*60 + "\n")
+    
+    # Batch Normalization
+    test_batch_normalization()
+    test_batch_normalization_4d()
+    test_batch_normalization_inference()
+    test_batch_norm_forward()
+    
+    # Layer Normalization
+    test_layer_normalization()
+    test_layer_normalization_3d()
+    test_layer_norm_forward()
+    
+    # Group Normalization
+    test_group_normalization()
+    test_group_normalization_invalid_groups()
+    
+    # Instance Normalization
+    test_instance_normalization()
+    
+    # Weight Normalization
+    test_weight_normalization()
+    test_weight_normalization_axis1()
+    
+    # Utilities
+    test_create_normalization_params()
+    test_apply_normalization()
+    test_apply_normalization_invalid()
+    
+    # Aliases
+    test_aliases()
+    
+    # Comparisons
+    test_batch_vs_layer_norm()
+    test_normalization_stability()
+    
+    print("\n" + "="*60)
+    print("ALL TESTS PASSED! ✓")
+    print("="*60 + "\n")
+
+
+if __name__ == "__main__":
+    run_all_tests()