noshot 11.0.0__py3-none-any.whl → 13.0.0__py3-none-any.whl

noshot/data/DLE FSD BDA/DLE/DLE 2 (tim stan s)/Lab Excercise (Training DNN).ipynb

@@ -0,0 +1,646 @@
+{
+ "cells": [
+  {
+   "cell_type": "raw",
+   "id": "fbc03c64-ef16-4473-b581-fc27a0fc5e37",
+   "metadata": {},
+   "source": [
+    "1. Print the pre-activated values and post-activated values of a neural network\n",
+    "2. Check the correctness of backpropagation\n",
+    "3. Standardise the values\n",
+    "4. Normalise the values by keeping min as 10 and max as 50\n",
+    "5. Initialize the weights using normal and uniform xavier initialization\n",
+    "5. Apply the variants of ReLU for the preactivated values and print the results\n",
+    "6. Apply exponential decay and inverse time decay for adjusting the initial learning rate\n",
+    "7. Implement a feed-forward neural network with different optimizers (SGD, Adagrad, RMSProp, RMSProp with Nesterov momentum, AdaDelta and Adam) and compare the results\n",
+    "8. Clip the gradients using value & norm based clipping\n",
+    "9. Implement Hessian-Free Optimization (HFO) by computing the Hessian-vector product (HVP) using PyTorch’s autograd, Tensorflow's tf.GradientTape (nested) and solve the Newton system Hv=−∇f using the Conjugate Gradient (CG) method.\n",
+    "10. Compute the Hessian matrix of a scalar function and determine if a critical point is a saddle point, local minimum, or local maximum by analyzing the eigenvalues of the Hessian"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "489759c2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "import torch\n",
+    "import torch.optim as optim\n",
+    "import tensorflow as tf\n",
+    "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
+    "from sklearn.datasets import make_regression\n",
+    "import matplotlib.pyplot as plt"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7e92ecaa",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "np.random.seed(42)\n",
+    "torch.manual_seed(42)\n",
+    "tf.random.set_seed(42)\n",
+    "\n",
+    "X = np.random.randn(10, 3)\n",
+    "y = np.random.randn(10, 1)"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "id": "d4c1f591-424f-44d0-bc84-c04f8839fc8f",
+   "metadata": {},
+   "source": [
+    "1. Print pre-activated and post-activated values"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7388227e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def print_activations(X, W, b, activation='relu'):\n",
+    "    pre_activation = np.dot(X, W) + b\n",
+    "    print(\"\\n1. Pre-activated values:\\n\", pre_activation)\n",
+    "    \n",
+    "    if activation == 'relu':\n",
+    "        post_activation = np.maximum(0, pre_activation)\n",
+    "    elif activation == 'sigmoid':\n",
+    "        post_activation = 1 / (1 + np.exp(-pre_activation))\n",
+    "    elif activation == 'tanh':\n",
+    "        post_activation = np.tanh(pre_activation)\n",
+    "    else:\n",
+    "        raise ValueError(\"Unsupported activation function\")\n",
+    "    \n",
+    "    print(\"\\nPost-activated values:\\n\", post_activation)\n",
+    "    return pre_activation, post_activation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6bcc03bf",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "W = np.random.randn(3, 4)\n",
+    "b = np.random.randn(4)\n",
+    "pre_act, post_act = print_activations(X, W, b, activation='relu')"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "id": "4cfb185b-a048-4806-879e-ad5af75b2299",
+   "metadata": {},
+   "source": [
+    "2. Check backpropagation correctness"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ca501ee2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def check_backprop():\n",
+    "    print(\"\\n2. Checking backpropagation...\")\n",
+    "    \n",
+    "    model = torch.nn.Sequential(\n",
+    "        torch.nn.Linear(2, 3),\n",
+    "        torch.nn.Tanh(),\n",
+    "        torch.nn.Linear(3, 1)\n",
+    "    ).double()\n",
+    "    \n",
+    "    X = torch.randn(5, 2, dtype=torch.double, requires_grad=True)\n",
+    "    y = torch.randn(5, 1, dtype=torch.double)\n",
+    "    \n",
+    "    test_input = torch.autograd.gradcheck(\n",
+    "        lambda x: torch.nn.functional.mse_loss(model(x), y),\n",
+    "        X,\n",
+    "        eps=1e-6,\n",
+    "        atol=1e-4,\n",
+    "        rtol=1e-4,\n",
+    "        raise_exception=False\n",
+    "    )\n",
+    "    print(\"Input gradient check passed:\", test_input)\n",
+    "    \n",
+    "    for name, param in model.named_parameters():\n",
+    "        if param.requires_grad:\n",
+    "            def func(input):\n",
+    "                with torch.no_grad():\n",
+    "                    old_data = param.data.clone()\n",
+    "                    param.data.copy_(input)\n",
+    "                output = model(X)\n",
+    "                loss = torch.nn.functional.mse_loss(output, y)\n",
+    "                with torch.no_grad():\n",
+    "                    param.data.copy_(old_data)\n",
+    "                return loss\n",
+    "            \n",
+    "            test_param = torch.autograd.gradcheck(\n",
+    "                func,\n",
+    "                param.data.clone().requires_grad_(True),\n",
+    "                eps=1e-6,\n",
+    "                atol=1e-4,\n",
+    "                rtol=1e-4,\n",
+    "                raise_exception=False\n",
+    "            )\n",
+    "            print(f\"Parameter {name} gradient check passed:\", test_param)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b2f0082f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "check_backprop()"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "id": "b5a4397a-fd32-4a34-a14d-8c777be04812",
+   "metadata": {},
+   "source": [
+    "3. Standardize values"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "75ec2b4b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def standardize_data(X):\n",
+    "    print(\"\\n3. Standardizing data...\")\n",
+    "    scaler = StandardScaler()\n",
+    "    X_std = scaler.fit_transform(X)\n",
+    "    print(\"Mean after standardization:\", X_std.mean(axis=0))\n",
+    "    print(\"Std after standardization:\", X_std.std(axis=0))\n",
+    "    return X_std"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "af896e94",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "X_std = standardize_data(X)"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "id": "116879b1-151c-4bcd-8ace-cab5d5daaa2f",
+   "metadata": {},
+   "source": [
+    "4. Normalize values (min=10, max=50)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4d77159d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def normalize_data(X):\n",
+    "    print(\"\\n4. Normalizing data (min=10, max=50)...\")\n",
+    "    scaler = MinMaxScaler(feature_range=(10, 50))\n",
+    "    X_norm = scaler.fit_transform(X)\n",
+    "    print(\"Min after normalization:\", X_norm.min(axis=0))\n",
+    "    print(\"Max after normalization:\", X_norm.max(axis=0))\n",
+    "    return X_norm"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d0d4596e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "X_norm = normalize_data(X)"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "id": "3e39dd60-9f56-48be-9d9f-2106a8209ea4",
+   "metadata": {},
+   "source": [
+    "5. Xavier initialization"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2132e8d4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def xavier_init(n_input, n_output):\n",
+    "    print(\"\\n5. Xavier initialization...\")\n",
+    "    std_normal = np.sqrt(2.0 / (n_input + n_output))\n",
+    "    weights_normal = np.random.normal(0, std_normal, (n_input, n_output))\n",
+    "    \n",
+    "    limit = np.sqrt(6.0 / (n_input + n_output))\n",
+    "    weights_uniform = np.random.uniform(-limit, limit, (n_input, n_output))\n",
+    "    \n",
+    "    print(\"Xavier Normal weights mean/std:\", weights_normal.mean(), weights_normal.std())\n",
+    "    print(\"Xavier Uniform weights min/max:\", weights_uniform.min(), weights_uniform.max())\n",
+    "    return weights_normal, weights_uniform"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "96b7c91b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "weights_normal, weights_uniform = xavier_init(3, 4)"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "id": "2273476f-e51c-48a0-9426-cf8624b6e963",
+   "metadata": {},
+   "source": [
+    "5. ReLU variants"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e4805aa6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def apply_relu_variants(pre_activation):\n",
+    "    print(\"\\n5. Applying ReLU variants...\")\n",
+    "    relu = np.maximum(0, pre_activation)\n",
+    "    \n",
+    "    leaky = np.where(pre_activation > 0, pre_activation, pre_activation * 0.01)\n",
+    "    \n",
+    "    parametric = np.where(pre_activation > 0, pre_activation, pre_activation * 0.25)\n",
+    "    \n",
+    "    elu = np.where(pre_activation > 0, pre_activation, 1.0 * (np.exp(pre_activation) - 1))\n",
+    "\n",
+    "    swish = pre_activation * (1 / (1 + np.exp(-pre_activation)))\n",
+    "    \n",
+    "    print(\"Original values:\\n\", pre_activation)\n",
+    "    print(\"\\nReLU:\\n\", relu)\n",
+    "    print(\"\\nLeaky ReLU:\\n\", leaky)\n",
+    "    print(\"\\nParametric ReLU:\\n\", parametric)\n",
+    "    print(\"\\nELU:\\n\", elu)\n",
+    "    print(\"\\nSwish:\\n\", swish)\n",
+    "    \n",
+    "    return {'relu': relu, 'leaky': leaky, 'parametric': parametric, 'elu': elu, 'swish': swish}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d91f44d1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "relu_results = apply_relu_variants(pre_act)"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "id": "6db6a2dc-1cf5-4f96-ba2b-e6d7ad219c38",
+   "metadata": {},
+   "source": [
+    "6. Learning rate decay"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f6903dab",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def learning_rate_decay(initial_lr=0.1, decay_type='exponential', steps=100):\n",
+    "    print(f\"\\n6. {decay_type} learning rate decay over {steps} steps:\")\n",
+    "    \n",
+    "    lr_values = []\n",
+    "    for global_step in range(steps):\n",
+    "        if decay_type == 'exponential':\n",
+    "            decay_rate = 0.96\n",
+    "            lr = initial_lr * decay_rate ** (global_step / (steps/10))\n",
+    "        elif decay_type == 'inverse_time':\n",
+    "            decay_rate = 1.0\n",
+    "            lr = initial_lr / (1 + decay_rate * global_step / (steps/10))\n",
+    "        else:\n",
+    "            raise ValueError(\"Unsupported decay type\")\n",
+    "        lr_values.append(lr)\n",
+    "    \n",
+    "    plt.figure(figsize=(10, 5))\n",
+    "    plt.plot(lr_values)\n",
+    "    plt.title(f\"{decay_type} Learning Rate Decay\")\n",
+    "    plt.xlabel(\"Step\")\n",
+    "    plt.ylabel(\"Learning Rate\")\n",
+    "    plt.grid()\n",
+    "    plt.show()\n",
+    "    \n",
+    "    return lr_values"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "759ef663",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "lr_exp = learning_rate_decay(decay_type='exponential')\n",
+    "lr_inv = learning_rate_decay(decay_type='inverse_time')"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "id": "865922ec-db01-4b34-b848-c1971f00ef62",
+   "metadata": {},
+   "source": [
+    "7. Neural network with different optimizers"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "75a65fab",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def compare_optimizers():\n",
+    "    print(\"\\n7. Comparing optimizers...\")\n",
+    "    \n",
+    "    X, y = make_regression(n_samples=100, n_features=5, noise=0.1, random_state=42)\n",
+    "    X = X.astype(np.float32)\n",
+    "    y = y.astype(np.float32).reshape(-1, 1)\n",
+    "\n",
+    "    X_tensor = torch.tensor(X, dtype=torch.float32)\n",
+    "    y_tensor = torch.tensor(y, dtype=torch.float32)\n",
+    "    \n",
+    "    def create_model():\n",
+    "        return torch.nn.Sequential(\n",
+    "            torch.nn.Linear(5, 10),\n",
+    "            torch.nn.ReLU(),\n",
+    "            torch.nn.Linear(10, 1)\n",
+    "        )\n",
+    "\n",
+    "    optimizers = {\n",
+    "        'SGD': optim.SGD,\n",
+    "        'Adagrad': optim.Adagrad,\n",
+    "        'RMSprop': optim.RMSprop,\n",
+    "        'Adam': optim.Adam,\n",
+    "        'Adadelta': optim.Adadelta\n",
+    "    }\n",
+    "    \n",
+    "    n_epochs = 200\n",
+    "    results = {}\n",
+    "    \n",
+    "    plt.figure(figsize=(12, 6))\n",
+    "    \n",
+    "    for name, opt_class in optimizers.items():\n",
+    "        model = create_model()\n",
+    "        optimizer = opt_class(model.parameters(), lr=0.01)\n",
+    "        \n",
+    "        losses = []\n",
+    "        for epoch in range(n_epochs):\n",
+    "            optimizer.zero_grad()\n",
+    "            y_pred = model(X_tensor)\n",
+    "            loss = torch.nn.functional.mse_loss(y_pred, y_tensor)\n",
+    "            loss.backward()\n",
+    "            optimizer.step()\n",
+    "            losses.append(loss.item())\n",
+    "        \n",
+    "        results[name] = losses\n",
+    "        plt.plot(losses, label=name)\n",
+    "        print(f\"{name}: Final loss = {losses[-1]:.4f}\")\n",
+    "    \n",
+    "    plt.title(\"Optimizer Comparison\")\n",
+    "    plt.xlabel(\"Epoch\")\n",
+    "    plt.ylabel(\"Loss\")\n",
+    "    plt.legend()\n",
+    "    plt.grid()\n",
+    "    plt.show()\n",
+    "    \n",
+    "    return results"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b09a56d5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "optimizer_results = compare_optimizers()"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "id": "4bbeda82-1dc3-4964-868e-2cce413cd927",
+   "metadata": {},
+   "source": [
+    "8. Gradient clipping"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7b281491",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def clip_gradients(gradients, method='value', threshold=1.0):\n",
+    "    print(f\"\\n8. Gradient clipping ({method} with threshold={threshold})...\")\n",
+    "    \n",
+    "    if method == 'value':\n",
+    "        clipped = [np.clip(g, -threshold, threshold) for g in gradients]\n",
+    "    elif method == 'norm':\n",
+    "        global_norm = np.sqrt(sum(np.sum(g**2) for g in gradients))\n",
+    "        scale = threshold / max(global_norm, threshold)\n",
+    "        clipped = [g * scale for g in gradients]\n",
+    "    else:\n",
+    "        raise ValueError(\"Unsupported clipping method\")\n",
+    "    \n",
+    "    print(\"Original gradient norms:\", [np.linalg.norm(g) for g in gradients])\n",
+    "    print(\"Clipped gradient norms:\", [np.linalg.norm(g) for g in clipped])\n",
+    "    \n",
+    "    return clipped"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c09dd387",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "gradients = [np.random.randn(3,4), np.random.randn(4,1)]\n",
+    "clipped_value = clip_gradients(gradients, method='value', threshold=1.0)\n",
+    "clipped_norm = clip_gradients(gradients, method='norm', threshold=1.0)"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "id": "6680e526-6710-481c-bccf-a0ab4b643eac",
+   "metadata": {},
+   "source": [
+    "9. Hessian-Free Optimization"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "500ee068",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def hessian_free_optimization():\n",
+    "    print(\"\\n9. Hessian-Free Optimization...\")\n",
+    "    \n",
+    "    print(\"PyTorch implementation:\")\n",
+    "    x = torch.randn(3, requires_grad=True, dtype=torch.double)\n",
+    "    \n",
+    "    def f(x):\n",
+    "        return x @ torch.diag(torch.tensor([1.0, 2.0, 3.0], dtype=torch.double)) @ x\n",
+    "    \n",
+    "    grad = torch.autograd.grad(f(x), x, create_graph=True)[0]\n",
+    "    \n",
+    "    def hvp(v):\n",
+    "        return torch.autograd.grad(grad @ v, x, retain_graph=True)[0]\n",
+    "    \n",
+    "    def cg_solve(A, b, max_iter=10, tol=1e-6):\n",
+    "        x = torch.zeros_like(b)\n",
+    "        r = b - A(x)\n",
+    "        p = r.clone()\n",
+    "        rsold = r @ r\n",
+    "        \n",
+    "        for i in range(max_iter):\n",
+    "            Ap = A(p)\n",
+    "            alpha = rsold / (p @ Ap)\n",
+    "            x = x + alpha * p\n",
+    "            r = r - alpha * Ap\n",
+    "            rsnew = r @ r\n",
+    "            if torch.sqrt(rsnew) < tol:\n",
+    "                break\n",
+    "            p = r + (rsnew / rsold) * p\n",
+    "            rsold = rsnew\n",
+    "        \n",
+    "        return x\n",
+    "    \n",
+    "    v = cg_solve(hvp, -grad)\n",
+    "    print(\"Solution v:\", v.detach().numpy())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5401b781",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "hessian_free_optimization()"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "id": "fe14e90e-d4a2-4d37-a27c-94b632a8c5a1",
+   "metadata": {},
+   "source": [
+    "10. Hessian matrix analysis"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cbc5032d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def hessian_analysis():\n",
+    "    print(\"\\n10. Hessian matrix analysis...\")\n",
+    "    \n",
+    "    def f(x):\n",
+    "        return x[0]**2 + x[1]**3 - x[1]**2\n",
+    "    \n",
+    "    critical_point = np.array([0, 2/3])\n",
+    "    \n",
+    "    eps = 1e-5\n",
+    "    hessian = np.zeros((2, 2))\n",
+    "    \n",
+    "    for i in range(2):\n",
+    "        for j in range(2):\n",
+    "            def partial_derivative(x):\n",
+    "                x_plus = x.copy()\n",
+    "                x_plus[j] += eps\n",
+    "                x_minus = x.copy()\n",
+    "                x_minus[j] -= eps\n",
+    "                return (f(x_plus) - f(x_minus)) / (2 * eps)\n",
+    "            \n",
+    "            x_plus = critical_point.copy()\n",
+    "            x_plus[i] += eps\n",
+    "            x_minus = critical_point.copy()\n",
+    "            x_minus[i] -= eps\n",
+    "            hessian[i,j] = (partial_derivative(x_plus) - partial_derivative(x_minus)) / (2 * eps)\n",
+    "    \n",
+    "    print(\"Hessian matrix:\\n\", hessian)\n",
+    "    \n",
+    "    eigenvalues = np.linalg.eigvals(hessian)\n",
+    "    print(\"Eigenvalues:\", eigenvalues)\n",
+    "    \n",
+    "    if all(eig > 0 for eig in eigenvalues):\n",
+    "        print(\"Conclusion: Local minimum\")\n",
+    "    elif all(eig < 0 for eig in eigenvalues):\n",
+    "        print(\"Conclusion: Local maximum\")\n",
+    "    else:\n",
+    "        print(\"Conclusion: Saddle point\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8a2503c5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "hessian_analysis()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}