noshot 0.1.7__py3-none-any.whl → 0.1.9__py3-none-any.whl

noshot/data/ML TS XAI/NLP/NLP 6/3. Naive Bayes WSD.ipynb

@@ -1,199 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "id": "26e01d66-ef30-485e-9881-6d81cfba8004",
-   "metadata": {},
-   "source": [
-    "# Experiment 3 :\n",
-    "<b>Implement decision rule-based Naïve Bayes disambiguation method to find the sense of an\r\n",
-    "ambiguous word with the given training set.</b>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 58,
-   "id": "81e97c34-fe56-467e-b7f0-a49732847a35",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import nltk\n",
-    "from nltk import word_tokenize\n",
-    "from nltk.corpus import stopwords\n",
-    "import string\n",
-    "import math\n",
-    "\n",
-    "#PRE-PROCESS THE SENTENCE by tokenizing and removing stopwords\n",
-    "def process(sentence) :\n",
-    "    sentence = sentence.translate(str.maketrans('', '', string.punctuation))\n",
-    "    word_tokens = word_tokenize(sentence)\n",
-    "    stops = set(stopwords.words('english'))\n",
-    "    return [word for word in word_tokens if word.lower() not in stops]\n",
-    "\n",
-    "with open('lab3.txt', 'r') as f :\n",
-    "    text = f.read()\n",
-    "dataset = {}\n",
-    "for data in text.split('\\n') :\n",
-    "    sentence, sense = data.split(':')\n",
-    "    if sense in dataset :\n",
-    "        dataset[sense].append(sentence)\n",
-    "    else :\n",
-    "        dataset[sense] = [sentence]\n",
-    "\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 59,
-   "id": "f76f3399-bb88-4dac-896d-b971756b9954",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "formatted_dataset = {sense : [] for sense in dataset}\n",
-    "for sense, sentences in dataset.items() :\n",
-    "    for sentence in sentences :\n",
-    "        formatted_dataset[sense] += process(sentence)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 60,
-   "id": "0286919d-2ef3-4b85-a3ff-9f4485976b77",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "vocabulary = {}\n",
-    "for words in formatted_dataset.values() :\n",
-    "    for word in words :\n",
-    "        if word in vocabulary :\n",
-    "            vocabulary[word] += 1\n",
-    "        else :\n",
-    "            vocabulary[word] = 1\n",
-    "                \n",
-    "priors = {sense : len(dataset[sense])/len(dataset) for sense in dataset}\n",
-    "\n",
-    "conditionals  = {}\n",
-    "for word in vocabulary :\n",
-    "    conditionals[word] = dict()\n",
-    "    for sense in dataset :\n",
-    "        count_wc = formatted_dataset[sense].count(word)\n",
-    "        count_c = len(dataset[sense])\n",
-    "        prop = (count_wc + 1) / (count_c + len(vocabulary))\n",
-    "        conditionals[word][sense] = prop\n",
-    "    "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 61,
-   "id": "4f5d059c-4fde-4655-b8d5-680132870a06",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def naive_bayes(sentence, priors, conditionals) :\n",
-    "    words = process(sentence)\n",
-    "    scores = {}\n",
-    "    for sense in priors :\n",
-    "        value = math.log(priors[sense])\n",
-    "        for word in words :\n",
-    "            if word in conditionals :\n",
-    "                value += math.log(conditionals[word][sense])\n",
-    "            else :\n",
-    "                value += math.log(1e-10)\n",
-    "        scores[sense] = value\n",
-    "\n",
-    "    sense_predicted = max(scores, key = scores.get)\n",
-    "    print(f\"{sentence}\\nSENSE : {sense_predicted}\\nSCORE : {scores[sense_predicted]}\")\n",
-    "    \n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 62,
-   "id": "adb227ad-78bb-454d-b0a4-5764014f2ed9",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "The light breeze was refreshing on a warm summer day.\n",
-      "SENSE :  Light (not intense)\n",
-      "SCORE : -25.59000306801089\n",
-      "She preferred light exercise, like walking and yoga.\n",
-      "SENSE :  Light (not intense)\n",
-      "SCORE : -26.113251211775438\n",
-      "This box is so light I can carry it with one hand.\n",
-      "SENSE :  Light (not heavy)\n",
-      "SCORE : -20.61710810189748\n",
-      "The light from the lamp was too bright for my eyes.\n",
-      "SENSE :  Light (brightness)\n",
-      "SCORE : -16.37372796051489\n",
-      "The light jacket was just perfect for the cool evening air.\n",
-      "SENSE :  Light (not heavy)\n",
-      "SCORE : -26.40093328422722\n",
-      "The light color of the walls made the room look bigger.\n",
-      "SENSE :  Light (color/appearance)\n",
-      "SCORE : -29.476708265454747\n",
-      "He gave a light chuckle when he heard the funny joke.\n",
-      "SENSE :  Light (mood/atmosphere)\n",
-      "SCORE : -27.078332107819023\n",
-      "I ordered a light meal because I wasn’t very hungry.\n",
-      "SENSE :  Light (not intense)\n",
-      "SCORE : -39.65089331873625\n",
-      "She wore a light dress for the summer party.\n",
-      "SENSE :  Light (color/appearance)\n",
-      "SCORE : -20.61710810189748\n",
-      "The light from the fireworks illuminated the night sky.\n",
-      "SENSE :  Light (brightness)\n",
-      "SCORE : -20.653475746068356\n"
-     ]
-    }
-   ],
-   "source": [
-    "test_data = ['The light breeze was refreshing on a warm summer day.',\n",
-    " 'She preferred light exercise, like walking and yoga.',\n",
-    " 'This box is so light I can carry it with one hand.',\n",
-    " 'The light from the lamp was too bright for my eyes.',\n",
-    " 'The light jacket was just perfect for the cool evening air.',\n",
-    " 'The light color of the walls made the room look bigger.',\n",
-    " 'He gave a light chuckle when he heard the funny joke.',\n",
-    " 'I ordered a light meal because I wasn’t very hungry.',\n",
-    " 'She wore a light dress for the summer party.',\n",
-    " 'The light from the fireworks illuminated the night sky.']\n",
-    "\n",
-    "for data in test_data :\n",
-    "    naive_bayes(data, priors, conditionals)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "c0e4a3b9-458b-47f0-9c39-c324116f24b3",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.11.5"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

noshot/data/ML TS XAI/NLP/NLP 6/4. Hinddle and Rooth.ipynb

@@ -1,151 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "id": "8b4fc6c3-60e4-4125-b0a0-64d3b83a7a85",
-   "metadata": {},
-   "source": [
-    "# Experiment 4 :\n",
-    "<b>Implement the Hindle and Rooth algorithm for solving the attachment ambiguity problem.</b>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 68,
-   "id": "1a6a36a7-daba-4c9f-8067-8262c924558c",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import nltk\n",
-    "from nltk import word_tokenize, sent_tokenize, pos_tag\n",
-    "from nltk.corpus import stopwords\n",
-    "import string\n",
-    "import math\n",
-    "\n",
-    "with open('lab4.txt', 'r') as f :\n",
-    "    text = f.read()\n",
-    "text = text.translate(str.maketrans('', '', string.punctuation))\n",
-    "text.replace('\\n', '')\n",
-    "tokens = word_tokenize(text)\n",
-    "tokens = [word for word in tokens if word.lower() not in ['a', 'an', 'the']]\n",
-    "bigrams = {}\n",
-    "for bigram in zip(tokens[:-1], tokens[1:]) :\n",
-    "    bigrams[bigram] = bigrams.get(bigram,0) + 1\n",
-    "\n",
-    "frequency = {}\n",
-    "for word in tokens :\n",
-    "    frequency[word] = frequency.get(word,0) + 1"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 61,
-   "id": "fc5414ff-9e7b-43b2-bc54-a7c564ffc3d4",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# use this to get a list of available text files\n",
-    "#print(gutenberg.fileids())"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 77,
-   "id": "2a46a595-3c13-4d70-96a3-578ed19be1bc",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def hindle_and_rooth(verb, prop, noun, frequency, bigrams) :\n",
-    "    nc = frequency.get(noun, 0)\n",
-    "    vc = frequency.get(verb, 0)\n",
-    "    pvc = bigrams.get((verb, prop), 0)\n",
-    "    pnc = bigrams.get((prop, noun), 0)\n",
-    "    P_VA = pvc/vc if vc > 0 else 0\n",
-    "    P_NA = pnc/nc if nc > 0 else 0\n",
-    "    print(P_VA, P_NA)\n",
-    "\n",
-    "    if not P_VA :\n",
-    "        print(f\"No occuurance of {prop} with {verb}\")\n",
-    "        print(f\"{prop} is attached with {noun} : Noun Attachment\")\n",
-    "        return\n",
-    "    if not P_NA :\n",
-    "        print(f\"No occuurance of {prop} with {noun}\")\n",
-    "        print(f\"{prop} is attached with {verb} : Verb Attachment\")\n",
-    "        return\n",
-    "\n",
-    "    try :\n",
-    "        x = (P_VA * (1 - P_NA))/P_NA\n",
-    "        lam = math.log2(x)\n",
-    "        if lam > 0 :\n",
-    "            print(f\"{prop} is attached with {verb} : Verb Attachment\")\n",
-    "        else :\n",
-    "            print(f\"{prop} is attached with {noun} : Noun Attachment\")\n",
-    "    except :\n",
-    "        print(f\"DATA INSUFFUCIENCY TO ARRIVE AT THE CONCLUSION\")\n",
-    "    finally :\n",
-    "        print(f\"occurance of noun in corpus : {nc}\")\n",
-    "        print(f\"occurance of verb in corpus : {vc}\")\n",
-    "        print(f\"occurance of verb and prop : {pvc}\")\n",
-    "        print(f\"occurance of noun and prop : {pnc}\")\n",
-    "        print(f\"Propability of noun attachment : {P_NA}\")\n",
-    "        print(f\"Propability of verb attachment : {P_VA}\")\n",
-    "        \n",
-    "        "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 78,
-   "id": "af0598a4-dfad-4c05-b575-3a1e1d74c5b0",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "0.3333333333333333 1.0\n",
-      "DATA INSUFFUCIENCY TO ARRIVE AT THE CONCLUSION\n",
-      "occurance of noun in corpus : 2\n",
-      "occurance of verb in corpus : 6\n",
-      "occurance of verb and prop : 2\n",
-      "occurance of noun and prop : 2\n",
-      "Propability of noun attachment : 1.0\n",
-      "Propability of verb attachment : 0.3333333333333333\n"
-     ]
-    }
-   ],
-   "source": [
-    "hindle_and_rooth('sat', 'on', 'bench', frequency, bigrams)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "dd37851f-b4e9-4a99-9d2e-8b909db292a2",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.11.5"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

noshot/data/ML TS XAI/NLP/NLP 6/5 and 6 FWD, BWD, Viterbi.ipynb

@@ -1,164 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "id": "8bc54aa3-b46b-4077-983b-6420776ec43f",
-   "metadata": {},
-   "source": [
-    "# Experiment 5 & 6 :\n",
-    "\n",
-    "<b>Implement forward and backward procedures using Hidden Markov Model to find the\n",
-    "probability of a word sequence given a hidden Markov model.</b>\n",
-    "\n",
-    "<b>Implement Viterbi algorithm to find the probability of a word sequence, and the best tag\n",
-    "sequence using Hidden Markov Model.</b>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 31,
-   "id": "b340e65d-d96d-49a1-bf9b-417eb908ad8a",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "[0.3, 0.4, 0.3]\n",
-      "[0.095, 0.175, 0.08000000000000002]\n",
-      "[0.021249999999999998, 0.06045, 0.0728]\n",
-      "[0.008334999999999999, 0.020425, 0.01209]\n",
-      "propability : 0.04085\n",
-      "[1, 1, 1]\n",
-      "[0.5, 0.5, 0.0]\n",
-      "[0.15, 0.12000000000000001, 0.06999999999999999]\n",
-      "[0.0675, 0.051500000000000004, 0.0]\n",
-      "propability : 0.040850000000000004\n"
-     ]
-    }
-   ],
-   "source": [
-    "states = ['CIA1', 'CIA2', 'CIA3']\n",
-    "prior = [0.3, 0.4, 0.3]\n",
-    "outputs = ['Good', 'Moderate', 'Poor']\n",
-    "emission = [\n",
-    "    [0.5, 0.3, 0.2],\n",
-    "    [0.5, 0.4, 0.1],\n",
-    "    [0.0, 0.7, 0.3]\n",
-    "]\n",
-    "transition = [\n",
-    "    [0.5, 0.5, 0.0],\n",
-    "    [0.1, 0.5, 0.4],\n",
-    "    [0.0, 0.2, 0.8]\n",
-    "]\n",
-    "seq = \"Good Moderate Good\"\n",
-    "seq = [outputs.index(x) for x in seq.split()]\n",
-    "\n",
-    "def HMMF(transition, emission, prior, seq) :\n",
-    "    alpha = prior\n",
-    "    states = len(alpha)\n",
-    "    for i in range(len(seq)) :\n",
-    "        print(alpha)\n",
-    "        new_alpha = [sum(transition[s1][s2]*emission[s1][seq[i]]*alpha[s1] for s1 in range(states)) for s2 in range(states)]\n",
-    "        alpha = new_alpha\n",
-    "    print(alpha)\n",
-    "    print(f\"propability : {sum(alpha)}\")\n",
-    "\n",
-    "def HMMB(transition, emission, prior, seq) :\n",
-    "    states = len(prior)\n",
-    "    beta = [1 for _ in range(states)]\n",
-    "    for i in range(len(seq)-1, -1, -1) :\n",
-    "        print(beta)\n",
-    "        new_beta = [sum(transition[s1][s2]*emission[s1][seq[i]]*beta[s2] for s2 in range(states)) for s1 in range(states)]\n",
-    "        beta = new_beta\n",
-    "    print(beta)\n",
-    "    print(f\"propability : {sum(beta[i]*prior[i] for i in range(states))}\")\n",
-    "\n",
-    "HMMF(transition, emission, prior, seq)\n",
-    "HMMB(transition, emission, prior, seq)\n",
-    "    "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 47,
-   "id": "ab7154e2-7a14-438f-b6f3-dbd256d6fe9d",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def viterbi(transition, emission, prior, seq, states) :\n",
-    "    delta = prior\n",
-    "    n = len(prior)\n",
-    "    psi = []\n",
-    "    for i in range(len(seq)) :\n",
-    "        print(delta)\n",
-    "        values = [\n",
-    "            [delta[s1]*transition[s1][s2]*emission[s1][seq[i]]for s1 in range(n)]\n",
-    "            for s2 in range(n)]\n",
-    "        new_delta = [max(value) for value in values]\n",
-    "        psi.append([values[i].index(new_delta[i]) for i in range(n)])\n",
-    "        delta = new_delta\n",
-    "    print(delta)\n",
-    "    bseq = []\n",
-    "    bseq.append(delta.index(max(delta)))\n",
-    "    for i in range(len(seq)-1, -1, -1) :\n",
-    "        val = psi[i][bseq[-1]]\n",
-    "        bseq.append(val)\n",
-    "\n",
-    "    print([states[i] for i in bseq[::-1]])\n",
-    "    "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 48,
-   "id": "aefd1ab7-cf20-4874-bad5-4dc61322a899",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "[0.3, 0.4, 0.3]\n",
-      "[0.075, 0.1, 0.08000000000000002]\n",
-      "[0.01125, 0.020000000000000004, 0.044800000000000006]\n",
-      "[0.0028125, 0.005000000000000001, 0.004000000000000001]\n",
-      "['CIA2', 'CIA2', 'CIA2', 'CIA2']\n"
-     ]
-    }
-   ],
-   "source": [
-    "viterbi(transition, emission, prior, seq, states)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "fd92c7ae-6d8f-42a7-a95d-5b4a4b4dda9c",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.11.5"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}