ilovetools 0.2.20__tar.gz → 0.2.21__tar.gz

{ilovetools-0.2.20/ilovetools.egg-info → ilovetools-0.2.21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ilovetools
-Version: 0.2.20
+Version: 0.2.21
 Summary: A comprehensive Python utility library with modular tools for AI/ML, data processing, and daily programming needs
 Home-page: https://github.com/AliMehdi512/ilovetools
 Author: Ali Mehdi
@@ -11,7 +11,7 @@ Project-URL: Repository, https://github.com/AliMehdi512/ilovetools
 Project-URL: Issues, https://github.com/AliMehdi512/ilovetools/issues
 Project-URL: Bug Reports, https://github.com/AliMehdi512/ilovetools/issues
 Project-URL: Source, https://github.com/AliMehdi512/ilovetools
-Keywords: utilities,tools,ai,ml,data-processing,automation,rnn,lstm,gru,recurrent-neural-networks,sequence-modeling,nlp
+Keywords: utilities,tools,ai,ml,data-processing,automation,batch-normalization,layer-normalization,normalization-techniques,deep-learning
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: Topic :: Software Development :: Libraries :: Python Modules

{ilovetools-0.2.20 → ilovetools-0.2.21}/ilovetools/__init__.py

@@ -2,8 +2,8 @@
 ilovetools - A comprehensive Python utility library
 """
 
-__version__ = "0.2.19"
-# release marker: 0.2.19
+__version__ = "0.2.20"
+# release marker: 0.2.20
 __author__ = "Ali Mehdi"
 __email__ = "ali.mehdi.dev579@gmail.com"
 

ilovetools-0.2.21/ilovetools/ml/normalization_advanced.py

@@ -0,0 +1,471 @@
+"""
+Advanced Normalization Techniques
+
+This module provides advanced normalization methods:
+- Batch Normalization (BatchNorm)
+- Layer Normalization (LayerNorm)
+- Instance Normalization (InstanceNorm)
+- Group Normalization (GroupNorm)
+- Weight Normalization
+- Spectral Normalization
+
+All operations support batched inputs and are optimized for deep learning.
+"""
+
+import numpy as np
+from typing import Tuple, Optional
+
+
+# ============================================================================
+# BATCH NORMALIZATION
+# ============================================================================
+
+def batch_norm_forward(
+    x: np.ndarray,
+    gamma: np.ndarray,
+    beta: np.ndarray,
+    running_mean: Optional[np.ndarray] = None,
+    running_var: Optional[np.ndarray] = None,
+    momentum: float = 0.9,
+    eps: float = 1e-5,
+    training: bool = True
+) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
+    """
+    Batch Normalization forward pass
+    
+    Normalizes across the batch dimension. Used in CNNs.
+    
+    Formula:
+        μ_B = (1/m) Σ x_i
+        σ²_B = (1/m) Σ (x_i - μ_B)²
+        x̂ = (x - μ_B) / √(σ²_B + ε)
+        y = γ x̂ + β
+    
+    Args:
+        x: Input tensor, shape (batch, channels, height, width) or (batch, features)
+        gamma: Scale parameter, shape (channels,) or (features,)
+        beta: Shift parameter, shape (channels,) or (features,)
+        running_mean: Running mean for inference
+        running_var: Running variance for inference
+        momentum: Momentum for running statistics
+        eps: Small constant for numerical stability
+        training: Whether in training mode
+        
+    Returns:
+        Tuple of (output, updated_running_mean, updated_running_var)
+        
+    Example:
+        >>> # For CNNs
+        >>> x = np.random.randn(32, 64, 28, 28)  # (batch, channels, H, W)
+        >>> gamma = np.ones(64)
+        >>> beta = np.zeros(64)
+        >>> output, mean, var = batch_norm_forward(x, gamma, beta, training=True)
+        >>> print(output.shape)  # (32, 64, 28, 28)
+        
+        >>> # For fully connected
+        >>> x = np.random.randn(32, 256)  # (batch, features)
+        >>> gamma = np.ones(256)
+        >>> beta = np.zeros(256)
+        >>> output, mean, var = batch_norm_forward(x, gamma, beta, training=True)
+        >>> print(output.shape)  # (32, 256)
+    """
+    if training:
+        # Calculate batch statistics
+        if x.ndim == 4:  # Conv: (batch, channels, height, width)
+            # Mean and variance across batch, height, width
+            axes = (0, 2, 3)
+            mean = np.mean(x, axis=axes, keepdims=True)
+            var = np.var(x, axis=axes, keepdims=True)
+            
+            # Squeeze for running stats
+            mean_squeeze = np.squeeze(mean, axis=(0, 2, 3))
+            var_squeeze = np.squeeze(var, axis=(0, 2, 3))
+        else:  # FC: (batch, features)
+            axes = 0
+            mean = np.mean(x, axis=axes, keepdims=True)
+            var = np.var(x, axis=axes, keepdims=True)
+            
+            mean_squeeze = np.squeeze(mean, axis=0)
+            var_squeeze = np.squeeze(var, axis=0)
+        
+        # Update running statistics
+        if running_mean is None:
+            running_mean = mean_squeeze
+        else:
+            running_mean = momentum * running_mean + (1 - momentum) * mean_squeeze
+            
+        if running_var is None:
+            running_var = var_squeeze
+        else:
+            running_var = momentum * running_var + (1 - momentum) * var_squeeze
+        
+        # Normalize
+        x_normalized = (x - mean) / np.sqrt(var + eps)
+    else:
+        # Use running statistics for inference
+        if running_mean is None or running_var is None:
+            raise ValueError("Running statistics required for inference mode")
+        
+        if x.ndim == 4:
+            mean = running_mean.reshape(1, -1, 1, 1)
+            var = running_var.reshape(1, -1, 1, 1)
+        else:
+            mean = running_mean.reshape(1, -1)
+            var = running_var.reshape(1, -1)
+        
+        x_normalized = (x - mean) / np.sqrt(var + eps)
+    
+    # Scale and shift
+    if x.ndim == 4:
+        gamma = gamma.reshape(1, -1, 1, 1)
+        beta = beta.reshape(1, -1, 1, 1)
+    else:
+        gamma = gamma.reshape(1, -1)
+        beta = beta.reshape(1, -1)
+    
+    output = gamma * x_normalized + beta
+    
+    return output, running_mean, running_var
+
+
+# ============================================================================
+# LAYER NORMALIZATION
+# ============================================================================
+
+def layer_norm_forward(
+    x: np.ndarray,
+    gamma: np.ndarray,
+    beta: np.ndarray,
+    eps: float = 1e-5
+) -> np.ndarray:
+    """
+    Layer Normalization forward pass
+    
+    Normalizes across the feature dimension. Used in Transformers and RNNs.
+    
+    Formula:
+        μ = (1/H) Σ x_i
+        σ² = (1/H) Σ (x_i - μ)²
+        x̂ = (x - μ) / √(σ² + ε)
+        y = γ x̂ + β
+    
+    Args:
+        x: Input tensor, shape (batch, features) or (batch, seq_len, features)
+        gamma: Scale parameter, shape (features,)
+        beta: Shift parameter, shape (features,)
+        eps: Small constant for numerical stability
+        
+    Returns:
+        Normalized output with same shape as input
+        
+    Example:
+        >>> # For Transformers
+        >>> x = np.random.randn(32, 10, 512)  # (batch, seq_len, features)
+        >>> gamma = np.ones(512)
+        >>> beta = np.zeros(512)
+        >>> output = layer_norm_forward(x, gamma, beta)
+        >>> print(output.shape)  # (32, 10, 512)
+        
+        >>> # For fully connected
+        >>> x = np.random.randn(32, 256)  # (batch, features)
+        >>> gamma = np.ones(256)
+        >>> beta = np.zeros(256)
+        >>> output = layer_norm_forward(x, gamma, beta)
+        >>> print(output.shape)  # (32, 256)
+    """
+    # Calculate mean and variance across feature dimension
+    if x.ndim == 3:  # (batch, seq_len, features)
+        axes = -1
+        mean = np.mean(x, axis=axes, keepdims=True)
+        var = np.var(x, axis=axes, keepdims=True)
+    elif x.ndim == 2:  # (batch, features)
+        axes = -1
+        mean = np.mean(x, axis=axes, keepdims=True)
+        var = np.var(x, axis=axes, keepdims=True)
+    else:
+        raise ValueError(f"Unsupported input shape: {x.shape}")
+    
+    # Normalize
+    x_normalized = (x - mean) / np.sqrt(var + eps)
+    
+    # Scale and shift
+    output = gamma * x_normalized + beta
+    
+    return output
+
+
+# ============================================================================
+# INSTANCE NORMALIZATION
+# ============================================================================
+
+def instance_norm_forward(
+    x: np.ndarray,
+    gamma: np.ndarray,
+    beta: np.ndarray,
+    eps: float = 1e-5
+) -> np.ndarray:
+    """
+    Instance Normalization forward pass
+    
+    Normalizes each sample and channel independently. Used in style transfer.
+    
+    Args:
+        x: Input tensor, shape (batch, channels, height, width)
+        gamma: Scale parameter, shape (channels,)
+        beta: Shift parameter, shape (channels,)
+        eps: Small constant for numerical stability
+        
+    Returns:
+        Normalized output with same shape as input
+        
+    Example:
+        >>> x = np.random.randn(32, 64, 28, 28)  # (batch, channels, H, W)
+        >>> gamma = np.ones(64)
+        >>> beta = np.zeros(64)
+        >>> output = instance_norm_forward(x, gamma, beta)
+        >>> print(output.shape)  # (32, 64, 28, 28)
+    """
+    if x.ndim != 4:
+        raise ValueError("Instance normalization requires 4D input (batch, channels, H, W)")
+    
+    # Calculate mean and variance per instance, per channel
+    # Normalize across spatial dimensions (H, W)
+    axes = (2, 3)
+    mean = np.mean(x, axis=axes, keepdims=True)
+    var = np.var(x, axis=axes, keepdims=True)
+    
+    # Normalize
+    x_normalized = (x - mean) / np.sqrt(var + eps)
+    
+    # Scale and shift
+    gamma = gamma.reshape(1, -1, 1, 1)
+    beta = beta.reshape(1, -1, 1, 1)
+    output = gamma * x_normalized + beta
+    
+    return output
+
+
+# ============================================================================
+# GROUP NORMALIZATION
+# ============================================================================
+
+def group_norm_forward(
+    x: np.ndarray,
+    gamma: np.ndarray,
+    beta: np.ndarray,
+    num_groups: int = 32,
+    eps: float = 1e-5
+) -> np.ndarray:
+    """
+    Group Normalization forward pass
+    
+    Divides channels into groups and normalizes within each group.
+    Good for small batch sizes.
+    
+    Args:
+        x: Input tensor, shape (batch, channels, height, width)
+        gamma: Scale parameter, shape (channels,)
+        beta: Shift parameter, shape (channels,)
+        num_groups: Number of groups to divide channels into
+        eps: Small constant for numerical stability
+        
+    Returns:
+        Normalized output with same shape as input
+        
+    Example:
+        >>> x = np.random.randn(32, 64, 28, 28)  # (batch, channels, H, W)
+        >>> gamma = np.ones(64)
+        >>> beta = np.zeros(64)
+        >>> output = group_norm_forward(x, gamma, beta, num_groups=32)
+        >>> print(output.shape)  # (32, 64, 28, 28)
+    """
+    if x.ndim != 4:
+        raise ValueError("Group normalization requires 4D input (batch, channels, H, W)")
+    
+    batch_size, channels, height, width = x.shape
+    
+    if channels % num_groups != 0:
+        raise ValueError(f"Number of channels ({channels}) must be divisible by num_groups ({num_groups})")
+    
+    # Reshape to separate groups
+    x_grouped = x.reshape(batch_size, num_groups, channels // num_groups, height, width)
+    
+    # Calculate mean and variance per group
+    axes = (2, 3, 4)
+    mean = np.mean(x_grouped, axis=axes, keepdims=True)
+    var = np.var(x_grouped, axis=axes, keepdims=True)
+    
+    # Normalize
+    x_normalized = (x_grouped - mean) / np.sqrt(var + eps)
+    
+    # Reshape back
+    x_normalized = x_normalized.reshape(batch_size, channels, height, width)
+    
+    # Scale and shift
+    gamma = gamma.reshape(1, -1, 1, 1)
+    beta = beta.reshape(1, -1, 1, 1)
+    output = gamma * x_normalized + beta
+    
+    return output
+
+
+# ============================================================================
+# WEIGHT NORMALIZATION
+# ============================================================================
+
+def weight_norm(
+    weight: np.ndarray,
+    dim: int = 0
+) -> Tuple[np.ndarray, np.ndarray]:
+    """
+    Weight Normalization
+    
+    Reparameterizes weight vectors to decouple magnitude and direction.
+    
+    Formula:
+        w = g * (v / ||v||)
+    
+    Args:
+        weight: Weight tensor
+        dim: Dimension along which to compute norm
+        
+    Returns:
+        Tuple of (normalized_weight, norm)
+        
+    Example:
+        >>> weight = np.random.randn(64, 128)  # (out_features, in_features)
+        >>> w_normalized, g = weight_norm(weight, dim=1)
+        >>> print(w_normalized.shape)  # (64, 128)
+        >>> print(g.shape)  # (64, 1)
+    """
+    # Calculate norm along specified dimension
+    norm = np.linalg.norm(weight, axis=dim, keepdims=True)
+    
+    # Normalize
+    normalized_weight = weight / (norm + 1e-8)
+    
+    return normalized_weight, norm
+
+
+# ============================================================================
+# SPECTRAL NORMALIZATION
+# ============================================================================
+
+def spectral_norm(
+    weight: np.ndarray,
+    num_iterations: int = 1
+) -> np.ndarray:
+    """
+    Spectral Normalization
+    
+    Normalizes weight matrix by its largest singular value.
+    Used in GANs for training stability.
+    
+    Args:
+        weight: Weight tensor, shape (out_features, in_features)
+        num_iterations: Number of power iterations
+        
+    Returns:
+        Spectrally normalized weight
+        
+    Example:
+        >>> weight = np.random.randn(64, 128)
+        >>> w_normalized = spectral_norm(weight, num_iterations=1)
+        >>> print(w_normalized.shape)  # (64, 128)
+    """
+    # Reshape weight to 2D if needed
+    original_shape = weight.shape
+    if weight.ndim > 2:
+        weight = weight.reshape(weight.shape[0], -1)
+    
+    # Power iteration to estimate largest singular value
+    u = np.random.randn(weight.shape[0])
+    u = u / np.linalg.norm(u)
+    
+    for _ in range(num_iterations):
+        v = np.dot(weight.T, u)
+        v = v / np.linalg.norm(v)
+        u = np.dot(weight, v)
+        u = u / np.linalg.norm(u)
+    
+    # Calculate spectral norm (largest singular value)
+    sigma = np.dot(u, np.dot(weight, v))
+    
+    # Normalize by spectral norm
+    normalized_weight = weight / sigma
+    
+    # Reshape back to original shape
+    normalized_weight = normalized_weight.reshape(original_shape)
+    
+    return normalized_weight
+
+
+# ============================================================================
+# UTILITY FUNCTIONS
+# ============================================================================
+
+def initialize_norm_params(
+    num_features: int
+) -> Tuple[np.ndarray, np.ndarray]:
+    """
+    Initialize normalization parameters
+    
+    Args:
+        num_features: Number of features/channels
+        
+    Returns:
+        Tuple of (gamma, beta)
+        - gamma: Scale parameter initialized to 1
+        - beta: Shift parameter initialized to 0
+        
+    Example:
+        >>> gamma, beta = initialize_norm_params(256)
+        >>> print(gamma.shape, beta.shape)  # (256,) (256,)
+    """
+    gamma = np.ones(num_features)
+    beta = np.zeros(num_features)
+    return gamma, beta
+
+
+def compute_norm_stats(
+    x: np.ndarray,
+    norm_type: str = 'batch'
+) -> Tuple[np.ndarray, np.ndarray]:
+    """
+    Compute normalization statistics
+    
+    Args:
+        x: Input tensor
+        norm_type: Type of normalization ('batch', 'layer', 'instance', 'group')
+        
+    Returns:
+        Tuple of (mean, variance)
+        
+    Example:
+        >>> x = np.random.randn(32, 64, 28, 28)
+        >>> mean, var = compute_norm_stats(x, norm_type='batch')
+        >>> print(mean.shape, var.shape)
+    """
+    if norm_type == 'batch':
+        if x.ndim == 4:
+            axes = (0, 2, 3)
+        else:
+            axes = 0
+    elif norm_type == 'layer':
+        axes = -1
+    elif norm_type == 'instance':
+        axes = (2, 3)
+    else:
+        raise ValueError(f"Unknown norm_type: {norm_type}")
+    
+    mean = np.mean(x, axis=axes, keepdims=True)
+    var = np.var(x, axis=axes, keepdims=True)
+    
+    return mean, var
+
+
+# Aliases for convenience
+batch_norm = batch_norm_forward
+layer_norm = layer_norm_forward
+instance_norm = instance_norm_forward
+group_norm = group_norm_forward

{ilovetools-0.2.20 → ilovetools-0.2.21/ilovetools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ilovetools
-Version: 0.2.20
+Version: 0.2.21
 Summary: A comprehensive Python utility library with modular tools for AI/ML, data processing, and daily programming needs
 Home-page: https://github.com/AliMehdi512/ilovetools
 Author: Ali Mehdi
@@ -11,7 +11,7 @@ Project-URL: Repository, https://github.com/AliMehdi512/ilovetools
 Project-URL: Issues, https://github.com/AliMehdi512/ilovetools/issues
 Project-URL: Bug Reports, https://github.com/AliMehdi512/ilovetools/issues
 Project-URL: Source, https://github.com/AliMehdi512/ilovetools
-Keywords: utilities,tools,ai,ml,data-processing,automation,rnn,lstm,gru,recurrent-neural-networks,sequence-modeling,nlp
+Keywords: utilities,tools,ai,ml,data-processing,automation,batch-normalization,layer-normalization,normalization-techniques,deep-learning
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: Topic :: Software Development :: Libraries :: Python Modules

{ilovetools-0.2.20 → ilovetools-0.2.21}/ilovetools.egg-info/SOURCES.txt

@@ -46,6 +46,7 @@ ilovetools/ml/loss_functions.py
 ilovetools/ml/metrics.py
 ilovetools/ml/neural_network.py
 ilovetools/ml/normalization.py
+ilovetools/ml/normalization_advanced.py
 ilovetools/ml/optimizers.py
 ilovetools/ml/pipeline.py
 ilovetools/ml/regularization.py
@@ -73,6 +74,7 @@ tests/test_gradient_descent.py
 tests/test_loss_functions.py
 tests/test_neural_network.py
 tests/test_normalization.py
+tests/test_normalization_advanced.py
 tests/test_optimizers.py
 tests/test_regularization.py
 tests/test_rnn.py

{ilovetools-0.2.20 → ilovetools-0.2.21}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ilovetools"
-version = "0.2.20"
+version = "0.2.21"
 description = "A comprehensive Python utility library with modular tools for AI/ML, data processing, and daily programming needs"
 readme = "README.md"
 requires-python = ">=3.8"
@@ -12,7 +12,7 @@ license = "MIT"
 authors = [
     {name = "Ali Mehdi", email = "ali.mehdi.dev579@gmail.com"}
 ]
-keywords = ["utilities", "tools", "ai", "ml", "data-processing", "automation", "rnn", "lstm", "gru", "recurrent-neural-networks", "sequence-modeling", "nlp"]
+keywords = ["utilities", "tools", "ai", "ml", "data-processing", "automation", "batch-normalization", "layer-normalization", "normalization-techniques", "deep-learning"]
 classifiers = [
     "Development Status :: 3 - Alpha",
     "Intended Audience :: Developers",

{ilovetools-0.2.20 → ilovetools-0.2.21}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="ilovetools",
-    version="0.2.20",
+    version="0.2.21",
     author="Ali Mehdi",
     author_email="ali.mehdi.dev579@gmail.com",
     description="A comprehensive Python utility library with modular tools for AI/ML, data processing, and daily programming needs",
@@ -57,7 +57,7 @@ setup(
             "soundfile>=0.12.0",
         ],
     },
-    keywords="utilities, tools, ai, ml, data-processing, automation, python-library, neural-networks, rnn, lstm, gru, recurrent-neural-networks, sequence-modeling, nlp",
+    keywords="utilities, tools, ai, ml, data-processing, automation, python-library, neural-networks, batch-normalization, layer-normalization, normalization-techniques, deep-learning",
     project_urls={
         "Bug Reports": "https://github.com/AliMehdi512/ilovetools/issues",
         "Source": "https://github.com/AliMehdi512/ilovetools",

ilovetools-0.2.21/tests/test_normalization_advanced.py

@@ -0,0 +1,375 @@
+"""
+Tests for advanced normalization techniques module
+"""
+
+import numpy as np
+import sys
+import os
+
+# Add parent directory to path
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+
+from ilovetools.ml.normalization_advanced import (
+    # Batch Normalization
+    batch_norm_forward,
+    # Layer Normalization
+    layer_norm_forward,
+    # Instance Normalization
+    instance_norm_forward,
+    # Group Normalization
+    group_norm_forward,
+    # Weight Normalization
+    weight_norm,
+    # Spectral Normalization
+    spectral_norm,
+    # Utilities
+    initialize_norm_params,
+    compute_norm_stats,
+    # Aliases
+    batch_norm,
+    layer_norm,
+    instance_norm,
+    group_norm,
+)
+
+
+def test_batch_norm_training():
+    """Test batch normalization in training mode"""
+    print("Testing batch_norm_forward (training)...")
+    
+    # Test with 4D input (CNN)
+    x = np.random.randn(32, 64, 28, 28)
+    gamma, beta = initialize_norm_params(64)
+    
+    output, running_mean, running_var = batch_norm_forward(
+        x, gamma, beta, training=True
+    )
+    
+    assert output.shape == x.shape, "Output shape should match input"
+    assert running_mean.shape == (64,), "Running mean shape incorrect"
+    assert running_var.shape == (64,), "Running var shape incorrect"
+    
+    # Check normalization (approximately zero mean, unit variance)
+    output_mean = np.mean(output, axis=(0, 2, 3))
+    output_var = np.var(output, axis=(0, 2, 3))
+    assert np.allclose(output_mean, 0, atol=1e-5), "Mean should be close to 0"
+    assert np.allclose(output_var, 1, atol=0.1), "Variance should be close to 1"
+    
+    print("✓ batch_norm_forward (training) passed")
+
+
+def test_batch_norm_inference():
+    """Test batch normalization in inference mode"""
+    print("Testing batch_norm_forward (inference)...")
+    
+    x = np.random.randn(32, 64, 28, 28)
+    gamma, beta = initialize_norm_params(64)
+    running_mean = np.zeros(64)
+    running_var = np.ones(64)
+    
+    output, _, _ = batch_norm_forward(
+        x, gamma, beta, running_mean, running_var, training=False
+    )
+    
+    assert output.shape == x.shape, "Output shape should match input"
+    
+    print("✓ batch_norm_forward (inference) passed")
+
+
+def test_batch_norm_fc():
+    """Test batch normalization with fully connected layers"""
+    print("Testing batch_norm_forward (FC)...")
+    
+    x = np.random.randn(32, 256)
+    gamma, beta = initialize_norm_params(256)
+    
+    output, running_mean, running_var = batch_norm_forward(
+        x, gamma, beta, training=True
+    )
+    
+    assert output.shape == x.shape, "Output shape should match input"
+    
+    # Check normalization
+    output_mean = np.mean(output, axis=0)
+    output_var = np.var(output, axis=0)
+    assert np.allclose(output_mean, 0, atol=1e-5), "Mean should be close to 0"
+    assert np.allclose(output_var, 1, atol=0.1), "Variance should be close to 1"
+    
+    print("✓ batch_norm_forward (FC) passed")
+
+
+def test_layer_norm_2d():
+    """Test layer normalization with 2D input"""
+    print("Testing layer_norm_forward (2D)...")
+    
+    x = np.random.randn(32, 256)
+    gamma, beta = initialize_norm_params(256)
+    
+    output = layer_norm_forward(x, gamma, beta)
+    
+    assert output.shape == x.shape, "Output shape should match input"
+    
+    # Check normalization per sample
+    for i in range(x.shape[0]):
+        sample_mean = np.mean(output[i])
+        sample_var = np.var(output[i])
+        assert np.isclose(sample_mean, 0, atol=1e-5), f"Sample {i} mean should be 0"
+        assert np.isclose(sample_var, 1, atol=0.1), f"Sample {i} variance should be 1"
+    
+    print("✓ layer_norm_forward (2D) passed")
+
+
+def test_layer_norm_3d():
+    """Test layer normalization with 3D input (Transformers)"""
+    print("Testing layer_norm_forward (3D)...")
+    
+    x = np.random.randn(32, 10, 512)  # (batch, seq_len, features)
+    gamma, beta = initialize_norm_params(512)
+    
+    output = layer_norm_forward(x, gamma, beta)
+    
+    assert output.shape == x.shape, "Output shape should match input"
+    
+    # Check normalization per sample, per timestep
+    for i in range(x.shape[0]):
+        for t in range(x.shape[1]):
+            sample_mean = np.mean(output[i, t])
+            sample_var = np.var(output[i, t])
+            assert np.isclose(sample_mean, 0, atol=1e-5), "Mean should be 0"
+            assert np.isclose(sample_var, 1, atol=0.1), "Variance should be 1"
+    
+    print("✓ layer_norm_forward (3D) passed")
+
+
+def test_instance_norm():
+    """Test instance normalization"""
+    print("Testing instance_norm_forward...")
+    
+    x = np.random.randn(32, 64, 28, 28)
+    gamma, beta = initialize_norm_params(64)
+    
+    output = instance_norm_forward(x, gamma, beta)
+    
+    assert output.shape == x.shape, "Output shape should match input"
+    
+    # Check normalization per instance, per channel
+    for i in range(x.shape[0]):
+        for c in range(x.shape[1]):
+            channel_mean = np.mean(output[i, c])
+            channel_var = np.var(output[i, c])
+            assert np.isclose(channel_mean, 0, atol=1e-5), "Mean should be 0"
+            assert np.isclose(channel_var, 1, atol=0.1), "Variance should be 1"
+    
+    print("✓ instance_norm_forward passed")
+
+
+def test_group_norm():
+    """Test group normalization"""
+    print("Testing group_norm_forward...")
+    
+    x = np.random.randn(32, 64, 28, 28)
+    gamma, beta = initialize_norm_params(64)
+    
+    output = group_norm_forward(x, gamma, beta, num_groups=32)
+    
+    assert output.shape == x.shape, "Output shape should match input"
+    
+    print("✓ group_norm_forward passed")
+
+
+def test_group_norm_invalid_groups():
+    """Test group normalization with invalid number of groups"""
+    print("Testing group_norm_forward with invalid groups...")
+    
+    x = np.random.randn(32, 64, 28, 28)
+    gamma, beta = initialize_norm_params(64)
+    
+    try:
+        output = group_norm_forward(x, gamma, beta, num_groups=7)  # 64 not divisible by 7
+        assert False, "Should raise ValueError"
+    except ValueError as e:
+        assert "divisible" in str(e).lower()
+    
+    print("✓ group_norm_forward invalid groups passed")
+
+
+def test_weight_norm():
+    """Test weight normalization"""
+    print("Testing weight_norm...")
+    
+    weight = np.random.randn(64, 128)
+    
+    w_normalized, g = weight_norm(weight, dim=1)
+    
+    assert w_normalized.shape == weight.shape, "Normalized weight shape should match"
+    assert g.shape == (64, 1), "Norm shape incorrect"
+    
+    # Check that each row has unit norm
+    row_norms = np.linalg.norm(w_normalized, axis=1)
+    assert np.allclose(row_norms, 1.0), "Each row should have unit norm"
+    
+    print("✓ weight_norm passed")
+
+
+def test_spectral_norm():
+    """Test spectral normalization"""
+    print("Testing spectral_norm...")
+    
+    weight = np.random.randn(64, 128)
+    
+    w_normalized = spectral_norm(weight, num_iterations=1)
+    
+    assert w_normalized.shape == weight.shape, "Normalized weight shape should match"
+    
+    # Check that largest singular value is approximately 1
+    u, s, v = np.linalg.svd(w_normalized, full_matrices=False)
+    assert np.isclose(s[0], 1.0, atol=0.1), "Largest singular value should be close to 1"
+    
+    print("✓ spectral_norm passed")
+
+
+def test_initialize_norm_params():
+    """Test normalization parameter initialization"""
+    print("Testing initialize_norm_params...")
+    
+    gamma, beta = initialize_norm_params(256)
+    
+    assert gamma.shape == (256,), "Gamma shape incorrect"
+    assert beta.shape == (256,), "Beta shape incorrect"
+    assert np.allclose(gamma, 1.0), "Gamma should be initialized to 1"
+    assert np.allclose(beta, 0.0), "Beta should be initialized to 0"
+    
+    print("✓ initialize_norm_params passed")
+
+
+def test_compute_norm_stats():
+    """Test normalization statistics computation"""
+    print("Testing compute_norm_stats...")
+    
+    x = np.random.randn(32, 64, 28, 28)
+    
+    # Batch norm stats
+    mean, var = compute_norm_stats(x, norm_type='batch')
+    assert mean.shape == (1, 64, 1, 1), "Batch norm mean shape incorrect"
+    assert var.shape == (1, 64, 1, 1), "Batch norm var shape incorrect"
+    
+    # Layer norm stats
+    mean, var = compute_norm_stats(x, norm_type='layer')
+    assert mean.shape == (32, 64, 28, 1), "Layer norm mean shape incorrect"
+    
+    # Instance norm stats
+    mean, var = compute_norm_stats(x, norm_type='instance')
+    assert mean.shape == (32, 64, 1, 1), "Instance norm mean shape incorrect"
+    
+    print("✓ compute_norm_stats passed")
+
+
+def test_aliases():
+    """Test function aliases"""
+    print("Testing aliases...")
+    
+    x = np.random.randn(32, 64, 28, 28)
+    gamma, beta = initialize_norm_params(64)
+    
+    # Test batch_norm alias
+    out1, _, _ = batch_norm(x, gamma, beta, training=True)
+    out2, _, _ = batch_norm_forward(x, gamma, beta, training=True)
+    assert np.allclose(out1, out2), "batch_norm alias should work"
+    
+    # Test layer_norm alias
+    x_2d = np.random.randn(32, 256)
+    gamma_2d, beta_2d = initialize_norm_params(256)
+    out1 = layer_norm(x_2d, gamma_2d, beta_2d)
+    out2 = layer_norm_forward(x_2d, gamma_2d, beta_2d)
+    assert np.allclose(out1, out2), "layer_norm alias should work"
+    
+    print("✓ aliases passed")
+
+
+def test_batch_norm_running_stats():
+    """Test batch normalization running statistics update"""
+    print("Testing batch_norm running statistics...")
+    
+    x = np.random.randn(32, 64, 28, 28)
+    gamma, beta = initialize_norm_params(64)
+    
+    running_mean = np.zeros(64)
+    running_var = np.ones(64)
+    
+    # First forward pass
+    _, running_mean_1, running_var_1 = batch_norm_forward(
+        x, gamma, beta, running_mean, running_var, momentum=0.9, training=True
+    )
+    
+    # Check that running stats were updated
+    assert not np.allclose(running_mean_1, running_mean), "Running mean should be updated"
+    assert not np.allclose(running_var_1, running_var), "Running var should be updated"
+    
+    print("✓ batch_norm running statistics passed")
+
+
+def test_normalization_with_scale_shift():
+    """Test that scale and shift parameters work correctly"""
+    print("Testing normalization with scale and shift...")
+    
+    x = np.random.randn(32, 256)
+    
+    # Custom gamma and beta
+    gamma = np.ones(256) * 2.0  # Scale by 2
+    beta = np.ones(256) * 3.0   # Shift by 3
+    
+    output = layer_norm_forward(x, gamma, beta)
+    
+    # After normalization, mean should be close to beta, std close to gamma
+    output_mean = np.mean(output, axis=0)
+    assert np.allclose(output_mean, 3.0, atol=0.1), "Mean should be close to beta"
+    
+    print("✓ normalization with scale and shift passed")
+
+
+def run_all_tests():
+    """Run all tests"""
+    print("\n" + "="*60)
+    print("ADVANCED NORMALIZATION TECHNIQUES MODULE TESTS")
+    print("="*60 + "\n")
+    
+    # Batch Normalization tests
+    test_batch_norm_training()
+    test_batch_norm_inference()
+    test_batch_norm_fc()
+    test_batch_norm_running_stats()
+    
+    # Layer Normalization tests
+    test_layer_norm_2d()
+    test_layer_norm_3d()
+    
+    # Instance Normalization tests
+    test_instance_norm()
+    
+    # Group Normalization tests
+    test_group_norm()
+    test_group_norm_invalid_groups()
+    
+    # Weight Normalization tests
+    test_weight_norm()
+    
+    # Spectral Normalization tests
+    test_spectral_norm()
+    
+    # Utility tests
+    test_initialize_norm_params()
+    test_compute_norm_stats()
+    
+    # Aliases
+    test_aliases()
+    
+    # Additional tests
+    test_normalization_with_scale_shift()
+    
+    print("\n" + "="*60)
+    print("ALL TESTS PASSED! ✓")
+    print("="*60 + "\n")
+
+
+if __name__ == "__main__":
+    run_all_tests()