noshot 0.1.0__py3-none-any.whl

noshot/data/AIDS CN NLP/NLP/NLP 6/3. Naive Bayes WSD.ipynb

@@ -0,0 +1,199 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "26e01d66-ef30-485e-9881-6d81cfba8004",
+   "metadata": {},
+   "source": [
+    "# Experiment 3 :\n",
+    "<b>Implement decision rule-based Naïve Bayes disambiguation method to find the sense of an\r\n",
+    "ambiguous word with the given training set.</b>"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 58,
+   "id": "81e97c34-fe56-467e-b7f0-a49732847a35",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import nltk\n",
+    "from nltk import word_tokenize\n",
+    "from nltk.corpus import stopwords\n",
+    "import string\n",
+    "import math\n",
+    "\n",
+    "#PRE-PROCESS THE SENTENCE by tokenizing and removing stopwords\n",
+    "def process(sentence) :\n",
+    "    sentence = sentence.translate(str.maketrans('', '', string.punctuation))\n",
+    "    word_tokens = word_tokenize(sentence)\n",
+    "    stops = set(stopwords.words('english'))\n",
+    "    return [word for word in word_tokens if word.lower() not in stops]\n",
+    "\n",
+    "with open('lab3.txt', 'r') as f :\n",
+    "    text = f.read()\n",
+    "dataset = {}\n",
+    "for data in text.split('\\n') :\n",
+    "    sentence, sense = data.split(':')\n",
+    "    if sense in dataset :\n",
+    "        dataset[sense].append(sentence)\n",
+    "    else :\n",
+    "        dataset[sense] = [sentence]\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 59,
+   "id": "f76f3399-bb88-4dac-896d-b971756b9954",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "formatted_dataset = {sense : [] for sense in dataset}\n",
+    "for sense, sentences in dataset.items() :\n",
+    "    for sentence in sentences :\n",
+    "        formatted_dataset[sense] += process(sentence)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 60,
+   "id": "0286919d-2ef3-4b85-a3ff-9f4485976b77",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "vocabulary = {}\n",
+    "for words in formatted_dataset.values() :\n",
+    "    for word in words :\n",
+    "        if word in vocabulary :\n",
+    "            vocabulary[word] += 1\n",
+    "        else :\n",
+    "            vocabulary[word] = 1\n",
+    "                \n",
+    "priors = {sense : len(dataset[sense])/len(dataset) for sense in dataset}\n",
+    "\n",
+    "conditionals  = {}\n",
+    "for word in vocabulary :\n",
+    "    conditionals[word] = dict()\n",
+    "    for sense in dataset :\n",
+    "        count_wc = formatted_dataset[sense].count(word)\n",
+    "        count_c = len(dataset[sense])\n",
+    "        prop = (count_wc + 1) / (count_c + len(vocabulary))\n",
+    "        conditionals[word][sense] = prop\n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 61,
+   "id": "4f5d059c-4fde-4655-b8d5-680132870a06",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def naive_bayes(sentence, priors, conditionals) :\n",
+    "    words = process(sentence)\n",
+    "    scores = {}\n",
+    "    for sense in priors :\n",
+    "        value = math.log(priors[sense])\n",
+    "        for word in words :\n",
+    "            if word in conditionals :\n",
+    "                value += math.log(conditionals[word][sense])\n",
+    "            else :\n",
+    "                value += math.log(1e-10)\n",
+    "        scores[sense] = value\n",
+    "\n",
+    "    sense_predicted = max(scores, key = scores.get)\n",
+    "    print(f\"{sentence}\\nSENSE : {sense_predicted}\\nSCORE : {scores[sense_predicted]}\")\n",
+    "    \n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 62,
+   "id": "adb227ad-78bb-454d-b0a4-5764014f2ed9",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The light breeze was refreshing on a warm summer day.\n",
+      "SENSE :  Light (not intense)\n",
+      "SCORE : -25.59000306801089\n",
+      "She preferred light exercise, like walking and yoga.\n",
+      "SENSE :  Light (not intense)\n",
+      "SCORE : -26.113251211775438\n",
+      "This box is so light I can carry it with one hand.\n",
+      "SENSE :  Light (not heavy)\n",
+      "SCORE : -20.61710810189748\n",
+      "The light from the lamp was too bright for my eyes.\n",
+      "SENSE :  Light (brightness)\n",
+      "SCORE : -16.37372796051489\n",
+      "The light jacket was just perfect for the cool evening air.\n",
+      "SENSE :  Light (not heavy)\n",
+      "SCORE : -26.40093328422722\n",
+      "The light color of the walls made the room look bigger.\n",
+      "SENSE :  Light (color/appearance)\n",
+      "SCORE : -29.476708265454747\n",
+      "He gave a light chuckle when he heard the funny joke.\n",
+      "SENSE :  Light (mood/atmosphere)\n",
+      "SCORE : -27.078332107819023\n",
+      "I ordered a light meal because I wasn’t very hungry.\n",
+      "SENSE :  Light (not intense)\n",
+      "SCORE : -39.65089331873625\n",
+      "She wore a light dress for the summer party.\n",
+      "SENSE :  Light (color/appearance)\n",
+      "SCORE : -20.61710810189748\n",
+      "The light from the fireworks illuminated the night sky.\n",
+      "SENSE :  Light (brightness)\n",
+      "SCORE : -20.653475746068356\n"
+     ]
+    }
+   ],
+   "source": [
+    "test_data = ['The light breeze was refreshing on a warm summer day.',\n",
+    " 'She preferred light exercise, like walking and yoga.',\n",
+    " 'This box is so light I can carry it with one hand.',\n",
+    " 'The light from the lamp was too bright for my eyes.',\n",
+    " 'The light jacket was just perfect for the cool evening air.',\n",
+    " 'The light color of the walls made the room look bigger.',\n",
+    " 'He gave a light chuckle when he heard the funny joke.',\n",
+    " 'I ordered a light meal because I wasn’t very hungry.',\n",
+    " 'She wore a light dress for the summer party.',\n",
+    " 'The light from the fireworks illuminated the night sky.']\n",
+    "\n",
+    "for data in test_data :\n",
+    "    naive_bayes(data, priors, conditionals)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c0e4a3b9-458b-47f0-9c39-c324116f24b3",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

noshot/data/AIDS CN NLP/NLP/NLP 6/4. Hinddle and Rooth.ipynb

@@ -0,0 +1,151 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "8b4fc6c3-60e4-4125-b0a0-64d3b83a7a85",
+   "metadata": {},
+   "source": [
+    "# Experiment 4 :\n",
+    "<b>Implement the Hindle and Rooth algorithm for solving the attachment ambiguity problem.</b>"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 68,
+   "id": "1a6a36a7-daba-4c9f-8067-8262c924558c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import nltk\n",
+    "from nltk import word_tokenize, sent_tokenize, pos_tag\n",
+    "from nltk.corpus import stopwords\n",
+    "import string\n",
+    "import math\n",
+    "\n",
+    "with open('lab4.txt', 'r') as f :\n",
+    "    text = f.read()\n",
+    "text = text.translate(str.maketrans('', '', string.punctuation))\n",
+    "text.replace('\\n', '')\n",
+    "tokens = word_tokenize(text)\n",
+    "tokens = [word for word in tokens if word.lower() not in ['a', 'an', 'the']]\n",
+    "bigrams = {}\n",
+    "for bigram in zip(tokens[:-1], tokens[1:]) :\n",
+    "    bigrams[bigram] = bigrams.get(bigram,0) + 1\n",
+    "\n",
+    "frequency = {}\n",
+    "for word in tokens :\n",
+    "    frequency[word] = frequency.get(word,0) + 1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 61,
+   "id": "fc5414ff-9e7b-43b2-bc54-a7c564ffc3d4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# use this to get a list of available text files\n",
+    "#print(gutenberg.fileids())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 77,
+   "id": "2a46a595-3c13-4d70-96a3-578ed19be1bc",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def hindle_and_rooth(verb, prop, noun, frequency, bigrams) :\n",
+    "    nc = frequency.get(noun, 0)\n",
+    "    vc = frequency.get(verb, 0)\n",
+    "    pvc = bigrams.get((verb, prop), 0)\n",
+    "    pnc = bigrams.get((prop, noun), 0)\n",
+    "    P_VA = pvc/vc if vc > 0 else 0\n",
+    "    P_NA = pnc/nc if nc > 0 else 0\n",
+    "    print(P_VA, P_NA)\n",
+    "\n",
+    "    if not P_VA :\n",
+    "        print(f\"No occuurance of {prop} with {verb}\")\n",
+    "        print(f\"{prop} is attached with {noun} : Noun Attachment\")\n",
+    "        return\n",
+    "    if not P_NA :\n",
+    "        print(f\"No occuurance of {prop} with {noun}\")\n",
+    "        print(f\"{prop} is attached with {verb} : Verb Attachment\")\n",
+    "        return\n",
+    "\n",
+    "    try :\n",
+    "        x = (P_VA * (1 - P_NA))/P_NA\n",
+    "        lam = math.log2(x)\n",
+    "        if lam > 0 :\n",
+    "            print(f\"{prop} is attached with {verb} : Verb Attachment\")\n",
+    "        else :\n",
+    "            print(f\"{prop} is attached with {noun} : Noun Attachment\")\n",
+    "    except :\n",
+    "        print(f\"DATA INSUFFUCIENCY TO ARRIVE AT THE CONCLUSION\")\n",
+    "    finally :\n",
+    "        print(f\"occurance of noun in corpus : {nc}\")\n",
+    "        print(f\"occurance of verb in corpus : {vc}\")\n",
+    "        print(f\"occurance of verb and prop : {pvc}\")\n",
+    "        print(f\"occurance of noun and prop : {pnc}\")\n",
+    "        print(f\"Propability of noun attachment : {P_NA}\")\n",
+    "        print(f\"Propability of verb attachment : {P_VA}\")\n",
+    "        \n",
+    "        "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 78,
+   "id": "af0598a4-dfad-4c05-b575-3a1e1d74c5b0",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "0.3333333333333333 1.0\n",
+      "DATA INSUFFUCIENCY TO ARRIVE AT THE CONCLUSION\n",
+      "occurance of noun in corpus : 2\n",
+      "occurance of verb in corpus : 6\n",
+      "occurance of verb and prop : 2\n",
+      "occurance of noun and prop : 2\n",
+      "Propability of noun attachment : 1.0\n",
+      "Propability of verb attachment : 0.3333333333333333\n"
+     ]
+    }
+   ],
+   "source": [
+    "hindle_and_rooth('sat', 'on', 'bench', frequency, bigrams)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "dd37851f-b4e9-4a99-9d2e-8b909db292a2",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

noshot/data/AIDS CN NLP/NLP/NLP 6/5 and 6 FWD, BWD, Viterbi.ipynb

@@ -0,0 +1,164 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "8bc54aa3-b46b-4077-983b-6420776ec43f",
+   "metadata": {},
+   "source": [
+    "# Experiment 5 & 6 :\n",
+    "\n",
+    "<b>Implement forward and backward procedures using Hidden Markov Model to find the\n",
+    "probability of a word sequence given a hidden Markov model.</b>\n",
+    "\n",
+    "<b>Implement Viterbi algorithm to find the probability of a word sequence, and the best tag\n",
+    "sequence using Hidden Markov Model.</b>"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "id": "b340e65d-d96d-49a1-bf9b-417eb908ad8a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[0.3, 0.4, 0.3]\n",
+      "[0.095, 0.175, 0.08000000000000002]\n",
+      "[0.021249999999999998, 0.06045, 0.0728]\n",
+      "[0.008334999999999999, 0.020425, 0.01209]\n",
+      "propability : 0.04085\n",
+      "[1, 1, 1]\n",
+      "[0.5, 0.5, 0.0]\n",
+      "[0.15, 0.12000000000000001, 0.06999999999999999]\n",
+      "[0.0675, 0.051500000000000004, 0.0]\n",
+      "propability : 0.040850000000000004\n"
+     ]
+    }
+   ],
+   "source": [
+    "states = ['CIA1', 'CIA2', 'CIA3']\n",
+    "prior = [0.3, 0.4, 0.3]\n",
+    "outputs = ['Good', 'Moderate', 'Poor']\n",
+    "emission = [\n",
+    "    [0.5, 0.3, 0.2],\n",
+    "    [0.5, 0.4, 0.1],\n",
+    "    [0.0, 0.7, 0.3]\n",
+    "]\n",
+    "transition = [\n",
+    "    [0.5, 0.5, 0.0],\n",
+    "    [0.1, 0.5, 0.4],\n",
+    "    [0.0, 0.2, 0.8]\n",
+    "]\n",
+    "seq = \"Good Moderate Good\"\n",
+    "seq = [outputs.index(x) for x in seq.split()]\n",
+    "\n",
+    "def HMMF(transition, emission, prior, seq) :\n",
+    "    alpha = prior\n",
+    "    states = len(alpha)\n",
+    "    for i in range(len(seq)) :\n",
+    "        print(alpha)\n",
+    "        new_alpha = [sum(transition[s1][s2]*emission[s1][seq[i]]*alpha[s1] for s1 in range(states)) for s2 in range(states)]\n",
+    "        alpha = new_alpha\n",
+    "    print(alpha)\n",
+    "    print(f\"propability : {sum(alpha)}\")\n",
+    "\n",
+    "def HMMB(transition, emission, prior, seq) :\n",
+    "    states = len(prior)\n",
+    "    beta = [1 for _ in range(states)]\n",
+    "    for i in range(len(seq)-1, -1, -1) :\n",
+    "        print(beta)\n",
+    "        new_beta = [sum(transition[s1][s2]*emission[s1][seq[i]]*beta[s2] for s2 in range(states)) for s1 in range(states)]\n",
+    "        beta = new_beta\n",
+    "    print(beta)\n",
+    "    print(f\"propability : {sum(beta[i]*prior[i] for i in range(states))}\")\n",
+    "\n",
+    "HMMF(transition, emission, prior, seq)\n",
+    "HMMB(transition, emission, prior, seq)\n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 47,
+   "id": "ab7154e2-7a14-438f-b6f3-dbd256d6fe9d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def viterbi(transition, emission, prior, seq, states) :\n",
+    "    delta = prior\n",
+    "    n = len(prior)\n",
+    "    psi = []\n",
+    "    for i in range(len(seq)) :\n",
+    "        print(delta)\n",
+    "        values = [\n",
+    "            [delta[s1]*transition[s1][s2]*emission[s1][seq[i]]for s1 in range(n)]\n",
+    "            for s2 in range(n)]\n",
+    "        new_delta = [max(value) for value in values]\n",
+    "        psi.append([values[i].index(new_delta[i]) for i in range(n)])\n",
+    "        delta = new_delta\n",
+    "    print(delta)\n",
+    "    bseq = []\n",
+    "    bseq.append(delta.index(max(delta)))\n",
+    "    for i in range(len(seq)-1, -1, -1) :\n",
+    "        val = psi[i][bseq[-1]]\n",
+    "        bseq.append(val)\n",
+    "\n",
+    "    print([states[i] for i in bseq[::-1]])\n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 48,
+   "id": "aefd1ab7-cf20-4874-bad5-4dc61322a899",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[0.3, 0.4, 0.3]\n",
+      "[0.075, 0.1, 0.08000000000000002]\n",
+      "[0.01125, 0.020000000000000004, 0.044800000000000006]\n",
+      "[0.0028125, 0.005000000000000001, 0.004000000000000001]\n",
+      "['CIA2', 'CIA2', 'CIA2', 'CIA2']\n"
+     ]
+    }
+   ],
+   "source": [
+    "viterbi(transition, emission, prior, seq, states)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fd92c7ae-6d8f-42a7-a95d-5b4a4b4dda9c",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}