noshot 0.1.0__py3-none-any.whl

noshot/data/AIDS CN NLP/NLP/NLP 2/Lab04(collocation)/Lab4-NLP-Exp-2.ipynb

@@ -0,0 +1,817 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "bac2330f",
+   "metadata": {},
+   "source": [
+    "Code credits: [**V Tarun Thothadri**](https://github.com/TarunThothadri)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "3969787f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import nltk\n",
+    "import re\n",
+    "from nltk.corpus import wordnet"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f4abdabf",
+   "metadata": {},
+   "source": [
+    "# Collocation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "675c4111",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "text = \"The computer is a miraculous scientific invention. It is a can store data and perform complex calculations in a very short period of time. Its primary characteristics are speed and accuracy. As a result, computers facilitate quick transactions and communication. It’s now common in schools, colleges, libraries, hospitals, offices, and banks. It is used in factories to save manual and mental labour as well as for quality control. A computer is a miraculous scientific invention. It is also used in various fields of research, such as space research. Computer courses at various levels are now being introduced in schools and colleges. The use of computers in specific fields is required for efficient service. Newspapers are the mirror of the world. It plays an important role in modern civilization. Newspapers are published in different languages ​​in our country. It helps us to gain common sense. Provides reviews and puzzles on newspaper reports, speeches, business, movies, sports, etc. Newspapers are read for both profit and pleasure. It brings to the public valuable perspectives, acts of injustice, oppression and maladministration. It angered the people and criticized the authorities for their failures. It forms public opinion. Therefore freedom of the press should not be hampered. Coal is a valuable hard, black material extracted from mines. Wood that has been buried for a long time becomes coal due to a chemical change. Earthquakes cause vast forest areas to sink underground and contribute to such changes as a result of tremendous heat and pressure. Coal mines can be found in our country at  Dhanbad, Jharia, Giridih, Chaibasa, and other locations. Coal is exported from India to Japan, Nepal, and Bangladesh. Coal is used as a fuel in both homes and factories and industries. The majority of trains and steamers move by burning coal in steam engines.\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "57bc2cd8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "R_patterns = [\n",
+    "   (r'won\\'t', 'will not'),\n",
+    "   (r'can\\'t', 'cannot'),\n",
+    "   (r'i\\'m', 'i am'),\n",
+    "   (r'(\\w+)\\'ll', '\\g<1> will'),\n",
+    "   (r'(\\w+)n\\'t', '\\g<1> not'),\n",
+    "   (r'(\\w+)\\'ve', '\\g<1> have'),\n",
+    "   (r'(\\w+)\\'s', '\\g<1> is'),\n",
+    "   (r'(\\w+)\\'re', '\\g<1> are'),\n",
+    "]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "23adb6ec",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class REReplacer(object):\n",
+    "   def __init__(self, pattern = R_patterns):\n",
+    "      self.pattern = [(re.compile(regex), repl) for (regex, repl) in pattern]\n",
+    "   def replace(self, text):\n",
+    "      s = text\n",
+    "      for (pattern, repl) in self.pattern:\n",
+    "         s = re.sub(pattern, repl, s)\n",
+    "      return s"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "0a0ee2ba",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "rep_word = REReplacer()\n",
+    "text = rep_word.replace(text)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 49,
+   "id": "e8833f53",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "text = text.replace(',','')\n",
+    "text = text.replace('.','')\n",
+    "text = text.replace('\\'','')\n",
+    "text = text.replace('\"','')\n",
+    "text = text.replace('’','')\n",
+    "text = text.replace('\\n','')\n",
+    "text = text.lower()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "id": "b6307deb",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['the',\n",
+       " 'computer',\n",
+       " 'is',\n",
+       " 'a',\n",
+       " 'miraculous',\n",
+       " 'scientific',\n",
+       " 'invention',\n",
+       " 'it',\n",
+       " 'is',\n",
+       " 'a',\n",
+       " 'can',\n",
+       " 'store',\n",
+       " 'data',\n",
+       " 'and',\n",
+       " 'perform',\n",
+       " 'complex',\n",
+       " 'calculations',\n",
+       " 'in',\n",
+       " 'a',\n",
+       " 'very',\n",
+       " 'short',\n",
+       " 'period',\n",
+       " 'of',\n",
+       " 'time',\n",
+       " 'its',\n",
+       " 'primary',\n",
+       " 'characteristics',\n",
+       " 'are',\n",
+       " 'speed',\n",
+       " 'and',\n",
+       " 'accuracy',\n",
+       " 'as',\n",
+       " 'a',\n",
+       " 'result',\n",
+       " 'computers',\n",
+       " 'facilitate',\n",
+       " 'quick',\n",
+       " 'transactions',\n",
+       " 'and',\n",
+       " 'communication',\n",
+       " 'its',\n",
+       " 'now',\n",
+       " 'common',\n",
+       " 'in',\n",
+       " 'schools',\n",
+       " 'colleges',\n",
+       " 'libraries',\n",
+       " 'hospitals',\n",
+       " 'offices',\n",
+       " 'and',\n",
+       " 'banks',\n",
+       " 'it',\n",
+       " 'is',\n",
+       " 'used',\n",
+       " 'in',\n",
+       " 'factories',\n",
+       " 'to',\n",
+       " 'save',\n",
+       " 'manual',\n",
+       " 'and',\n",
+       " 'mental',\n",
+       " 'labour',\n",
+       " 'as',\n",
+       " 'well',\n",
+       " 'as',\n",
+       " 'for',\n",
+       " 'quality',\n",
+       " 'control',\n",
+       " 'a',\n",
+       " 'computer',\n",
+       " 'is',\n",
+       " 'a',\n",
+       " 'miraculous',\n",
+       " 'scientific',\n",
+       " 'invention',\n",
+       " 'it',\n",
+       " 'is',\n",
+       " 'also',\n",
+       " 'used',\n",
+       " 'in',\n",
+       " 'various',\n",
+       " 'fields',\n",
+       " 'of',\n",
+       " 'research',\n",
+       " 'such',\n",
+       " 'as',\n",
+       " 'space',\n",
+       " 'research',\n",
+       " 'computer',\n",
+       " 'courses',\n",
+       " 'at',\n",
+       " 'various',\n",
+       " 'levels',\n",
+       " 'are',\n",
+       " 'now',\n",
+       " 'being',\n",
+       " 'introduced',\n",
+       " 'in',\n",
+       " 'schools',\n",
+       " 'and',\n",
+       " 'colleges',\n",
+       " 'the',\n",
+       " 'use',\n",
+       " 'of',\n",
+       " 'computers',\n",
+       " 'in',\n",
+       " 'specific',\n",
+       " 'fields',\n",
+       " 'is',\n",
+       " 'required',\n",
+       " 'for',\n",
+       " 'efficient',\n",
+       " 'service',\n",
+       " 'newspapers',\n",
+       " 'are',\n",
+       " 'the',\n",
+       " 'mirror',\n",
+       " 'of',\n",
+       " 'the',\n",
+       " 'world',\n",
+       " 'it',\n",
+       " 'plays',\n",
+       " 'an',\n",
+       " 'important',\n",
+       " 'role',\n",
+       " 'in',\n",
+       " 'modern',\n",
+       " 'civilization',\n",
+       " 'newspapers',\n",
+       " 'are',\n",
+       " 'published',\n",
+       " 'in',\n",
+       " 'different',\n",
+       " 'languages',\n",
+       " '\\u200b\\u200bin',\n",
+       " 'our',\n",
+       " 'country',\n",
+       " 'it',\n",
+       " 'helps',\n",
+       " 'us',\n",
+       " 'to',\n",
+       " 'gain',\n",
+       " 'common',\n",
+       " 'sense',\n",
+       " 'provides',\n",
+       " 'reviews',\n",
+       " 'and',\n",
+       " 'puzzles',\n",
+       " 'on',\n",
+       " 'newspaper',\n",
+       " 'reports',\n",
+       " 'speeches',\n",
+       " 'business',\n",
+       " 'movies',\n",
+       " 'sports',\n",
+       " 'etc',\n",
+       " 'newspapers',\n",
+       " 'are',\n",
+       " 'read',\n",
+       " 'for',\n",
+       " 'both',\n",
+       " 'profit',\n",
+       " 'and',\n",
+       " 'pleasure',\n",
+       " 'it',\n",
+       " 'brings',\n",
+       " 'to',\n",
+       " 'the',\n",
+       " 'public',\n",
+       " 'valuable',\n",
+       " 'perspectives',\n",
+       " 'acts',\n",
+       " 'of',\n",
+       " 'injustice',\n",
+       " 'oppression',\n",
+       " 'and',\n",
+       " 'maladministration',\n",
+       " 'it',\n",
+       " 'angered',\n",
+       " 'the',\n",
+       " 'people',\n",
+       " 'and',\n",
+       " 'criticized',\n",
+       " 'the',\n",
+       " 'authorities',\n",
+       " 'for',\n",
+       " 'their',\n",
+       " 'failures',\n",
+       " 'it',\n",
+       " 'forms',\n",
+       " 'public',\n",
+       " 'opinion',\n",
+       " 'therefore',\n",
+       " 'freedom',\n",
+       " 'of',\n",
+       " 'the',\n",
+       " 'press',\n",
+       " 'should',\n",
+       " 'not',\n",
+       " 'be',\n",
+       " 'hampered',\n",
+       " 'coal',\n",
+       " 'is',\n",
+       " 'a',\n",
+       " 'valuable',\n",
+       " 'hard',\n",
+       " 'black',\n",
+       " 'material',\n",
+       " 'extracted',\n",
+       " 'from',\n",
+       " 'mines',\n",
+       " 'wood',\n",
+       " 'that',\n",
+       " 'has',\n",
+       " 'been',\n",
+       " 'buried',\n",
+       " 'for',\n",
+       " 'a',\n",
+       " 'long',\n",
+       " 'time',\n",
+       " 'becomes',\n",
+       " 'coal',\n",
+       " 'due',\n",
+       " 'to',\n",
+       " 'a',\n",
+       " 'chemical',\n",
+       " 'change',\n",
+       " 'earthquakes',\n",
+       " 'cause',\n",
+       " 'vast',\n",
+       " 'forest',\n",
+       " 'areas',\n",
+       " 'to',\n",
+       " 'sink',\n",
+       " 'underground',\n",
+       " 'and',\n",
+       " 'contribute',\n",
+       " 'to',\n",
+       " 'such',\n",
+       " 'changes',\n",
+       " 'as',\n",
+       " 'a',\n",
+       " 'result',\n",
+       " 'of',\n",
+       " 'tremendous',\n",
+       " 'heat',\n",
+       " 'and',\n",
+       " 'pressure',\n",
+       " 'coal',\n",
+       " 'mines',\n",
+       " 'can',\n",
+       " 'be',\n",
+       " 'found',\n",
+       " 'in',\n",
+       " 'our',\n",
+       " 'country',\n",
+       " 'at',\n",
+       " 'dhanbad',\n",
+       " 'jharia',\n",
+       " 'giridih',\n",
+       " 'chaibasa',\n",
+       " 'and',\n",
+       " 'other',\n",
+       " 'locations',\n",
+       " 'coal',\n",
+       " 'is',\n",
+       " 'exported',\n",
+       " 'from',\n",
+       " 'india',\n",
+       " 'to',\n",
+       " 'japan',\n",
+       " 'nepal',\n",
+       " 'and',\n",
+       " 'bangladesh',\n",
+       " 'coal',\n",
+       " 'is',\n",
+       " 'used',\n",
+       " 'as',\n",
+       " 'a',\n",
+       " 'fuel',\n",
+       " 'in',\n",
+       " 'both',\n",
+       " 'homes',\n",
+       " 'and',\n",
+       " 'factories',\n",
+       " 'and',\n",
+       " 'industries',\n",
+       " 'the',\n",
+       " 'majority',\n",
+       " 'of',\n",
+       " 'trains',\n",
+       " 'and',\n",
+       " 'steamers',\n",
+       " 'move',\n",
+       " 'by',\n",
+       " 'burning',\n",
+       " 'coal',\n",
+       " 'in',\n",
+       " 'steam',\n",
+       " 'engines']"
+      ]
+     },
+     "execution_count": 33,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from nltk.tokenize import TreebankWordTokenizer\n",
+    "tokenizer_wrd = TreebankWordTokenizer()\n",
+    "tokenized = tokenizer_wrd.tokenize(text)\n",
+    "tokenized"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "10cac8f7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "d = {}\n",
+    "for word in tokenized:\n",
+    "    if word not in d:\n",
+    "        d[word] = 1\n",
+    "    else:\n",
+    "        d[word] += 1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 37,
+   "id": "aa070a0a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The number of words :  300\n",
+      "The number of unique words :  178\n",
+      "Frequencies of the words.....\n",
+      " \n"
+     ]
+    }
+   ],
+   "source": [
+    "nuws = len(d)\n",
+    "nws = len(tokenized)\n",
+    "print(\"The number of words : \",str(nws))\n",
+    "print(\"The number of unique words : \",str(nuws))\n",
+    "print(\"Frequencies of the words.....\")\n",
+    "print(\" \")\n",
+    "\n",
+    "#for i in d:\n",
+    "#    print(i,\" \",str(d[i]))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 38,
+   "id": "5a2a0e98",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Maximum probability bigram pair ....\n",
+      "is a   4\n",
+      "it is   3\n",
+      "as a   3\n"
+     ]
+    }
+   ],
+   "source": [
+    "#Bigram\n",
+    "jd = {}\n",
+    "for i in range(len(tokenized)):\n",
+    "    w1 = i\n",
+    "    w2 = i+1\n",
+    "    if(w2 != len(tokenized)):\n",
+    "        count = 0\n",
+    "        wd1 = tokenized[w1]\n",
+    "        wd2 = tokenized[w2]\n",
+    "        for j in range(len(tokenized)):\n",
+    "            if(tokenized[j] == wd1 and tokenized[j+1] == wd2):\n",
+    "                count += 1\n",
+    "        jd[str(wd1 + \" \" + wd2)] = count\n",
+    "'''for i in jd:\n",
+    "    print(\"Joint probability of \",i,\" : \",str(jd[i]/nws))'''\n",
+    "    \n",
+    "maxp = 0\n",
+    "maxq = 0\n",
+    "maxr = 0\n",
+    "for i in jd:\n",
+    "    if jd[i] > maxp:\n",
+    "        maxp = jd[i]\n",
+    "    elif jd[i] >maxq and jd[i]<= maxp:\n",
+    "        maxq = jd[i]\n",
+    "    elif jd[i] > maxr and jd[i] <= maxr and jd[i] <= maxp:\n",
+    "        maxr = jd[i]\n",
+    "print(\"Maximum probability bigram pair ....\")\n",
+    "j = 0\n",
+    "top_3 = []\n",
+    "for i in jd:\n",
+    "    if jd[i] == maxp:\n",
+    "        print(i,' ',str(jd[i]))\n",
+    "        top_3.append(i)\n",
+    "        j += 1\n",
+    "        if(j>=3):\n",
+    "            break\n",
+    "    elif jd[i] == maxq:\n",
+    "        print(i,' ',str(jd[i]))\n",
+    "        top_3.append(i)\n",
+    "        j += 1\n",
+    "        if(j>=3):\n",
+    "            break\n",
+    "    elif jd[i] == maxr:\n",
+    "        print(i,' ',str(jd[i]))\n",
+    "        top_3.append(i)\n",
+    "        j += 1\n",
+    "        if(j>=3):\n",
+    "            break"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 39,
+   "id": "6ead6a26",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Maximum probability trigram pair ....\n",
+      "computer is a   2\n",
+      "is a miraculous   2\n",
+      "a miraculous scientific   2\n",
+      "miraculous scientific invention   2\n",
+      "scientific invention it   2\n",
+      "invention it is   2\n",
+      "as a result   2\n"
+     ]
+    }
+   ],
+   "source": [
+    "#trigram\n",
+    "td = {}\n",
+    "for i in range(len(tokenized)):\n",
+    "    w1 = i\n",
+    "    w2 = i+1\n",
+    "    w3 = i+2\n",
+    "    if(w2!= len(tokenized) and w3 != len(tokenized)):\n",
+    "        count = 0\n",
+    "        wd1 = tokenized[w1]\n",
+    "        wd2 = tokenized[w2]\n",
+    "        wd3 = tokenized[w3]\n",
+    "        for j in range(len(tokenized)):\n",
+    "            if(tokenized[j] == wd1 and tokenized[j+1] == wd2 and tokenized[j+2] == wd3):\n",
+    "                count += 1\n",
+    "        td[str(wd1 + \" \" + wd2 + \" \" + wd3)] = count\n",
+    "'''for i in td:\n",
+    "    print(\"Joint probability of \",i,\" : \",str(td[i]/nws))'''\n",
+    "\n",
+    "    \n",
+    "maxp = 0\n",
+    "for i in td:\n",
+    "    if td[i] > maxp:\n",
+    "        maxp = td[i]\n",
+    "print(\"Maximum probability trigram pair ....\")\n",
+    "for i in td:\n",
+    "    if td[i] == maxp:\n",
+    "        print(i,' ',str(td[i]))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 40,
+   "id": "e44a7339",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Maximum probability 4gram pair ....\n",
+      "computer is a miraculous   2\n",
+      "is a miraculous scientific   2\n",
+      "a miraculous scientific invention   2\n",
+      "miraculous scientific invention it   2\n",
+      "scientific invention it is   2\n"
+     ]
+    }
+   ],
+   "source": [
+    "#4gram\n",
+    "fd = {}\n",
+    "for i in range(len(tokenized)):\n",
+    "    w1 = i\n",
+    "    w2 = i+1\n",
+    "    w3 = i+2\n",
+    "    w4 = i+3\n",
+    "    if(w2!= len(tokenized) and w3 != len(tokenized) and w3 != len(tokenized) and w4 != len(tokenized)):\n",
+    "        count = 0\n",
+    "        wd1 = tokenized[w1]\n",
+    "        wd2 = tokenized[w2]\n",
+    "        wd3 = tokenized[w3]\n",
+    "        wd4 = tokenized[w4]\n",
+    "        for j in range(len(tokenized)):\n",
+    "            if(tokenized[j] == wd1 and tokenized[j+1] == wd2 and tokenized[j+2] == wd3 and tokenized[j+3] == wd4):\n",
+    "                count += 1\n",
+    "        fd[str(wd1 + \" \" + wd2 + \" \" + wd3 + \" \" + wd4)] = count\n",
+    "'''for i in fd:\n",
+    "    print(\"Joint probability of \",i,\" : \",str(fd[i]/nws))'''\n",
+    "\n",
+    "    \n",
+    "maxp = 0\n",
+    "for i in fd:\n",
+    "    if fd[i] > maxp:\n",
+    "        maxp = fd[i]\n",
+    "print(\"Maximum probability 4gram pair ....\")\n",
+    "for i in fd:\n",
+    "    if fd[i] == maxp:\n",
+    "        print(i,' ',str(fd[i]))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 41,
+   "id": "ae199e34",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Maximum probability 4gram pair ....\n",
+      "computer is a miraculous scientific   2\n",
+      "is a miraculous scientific invention   2\n",
+      "a miraculous scientific invention it   2\n",
+      "miraculous scientific invention it is   2\n"
+     ]
+    }
+   ],
+   "source": [
+    "#5gram\n",
+    "fid = {}\n",
+    "for i in range(len(tokenized)):\n",
+    "    w1 = i\n",
+    "    w2 = i+1\n",
+    "    w3 = i+2\n",
+    "    w4 = i+3\n",
+    "    w5 = i+4\n",
+    "    if(w2!= len(tokenized) and w3 != len(tokenized) and w3 != len(tokenized) and w4 != len(tokenized) and w5 != len(tokenized)):\n",
+    "        count = 0\n",
+    "        wd1 = tokenized[w1]\n",
+    "        wd2 = tokenized[w2]\n",
+    "        wd3 = tokenized[w3]\n",
+    "        wd4 = tokenized[w4]\n",
+    "        wd5 = tokenized[w5]\n",
+    "        for j in range(len(tokenized)):\n",
+    "            if(tokenized[j] == wd1 and tokenized[j+1] == wd2 and tokenized[j+2] == wd3 and tokenized[j+3] == wd4 and tokenized[j+4] == wd5):\n",
+    "                count += 1\n",
+    "        fid[str(wd1 + \" \" + wd2 + \" \" + wd3 + \" \" + wd4 + \" \" + wd5)] = count\n",
+    "'''for i in fid:\n",
+    "    print(\"Joint probability of \",i,\" : \",str(fid[i]/nws))'''\n",
+    "\n",
+    "    \n",
+    "maxp = 0\n",
+    "for i in fid:\n",
+    "    if fid[i] > maxp:\n",
+    "        maxp = fid[i]\n",
+    "print(\"Maximum probability 4gram pair ....\")\n",
+    "for i in fid:\n",
+    "    if fid[i] == maxp:\n",
+    "        print(i,' ',str(fid[i]))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c2facedf",
+   "metadata": {},
+   "source": [
+    "# t - test"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 42,
+   "id": "53dde70e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The t score for 'is a' is : 1.8473570419766019\n",
+      "The t score for 'it is' is : 1.60151443479242\n",
+      "The t score for 'as a' is : 1.613119611856133\n"
+     ]
+    }
+   ],
+   "source": [
+    "for i in top_3:\n",
+    "    wrd1 = i.split(' ')[0]\n",
+    "    wrd2 = i.split(' ')[1]\n",
+    "    p_wrd1 = float(d[wrd1]/nws)\n",
+    "    p_wrd2 = float(d[wrd2]/nws)\n",
+    "    mu = float(p_wrd1*p_wrd2)\n",
+    "    mean = float(jd[i]/nws)\n",
+    "    s_sq = float(mean*float(1-mean))\n",
+    "    t_numerator = float(mean - mu)\n",
+    "    t_denominator = float((s_sq/nws)**(0.5))\n",
+    "    t_score = float(t_numerator/t_denominator)\n",
+    "    print(\"The t score for '\"+i+\"' is : \"+str(t_score))\n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0c723ee4",
+   "metadata": {},
+   "source": [
+    "# Pearson's chi-square test"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 48,
+   "id": "efae5df1",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Observed :  4 7 5 284\n",
+      "Expected :  0.33 10.67 8.67 280.33\n",
+      "X^2      :  40.81484848484848 1.262314901593252 1.5535063437139562 0.04804658794991659\n",
+      "The chi-square score for 'is a' is : 43.67871631810561\n",
+      "Observed :  3 6 5 286\n",
+      "Expected :  0.24 8.76 7.76 283.24\n",
+      "X^2      :  31.739999999999995 0.8695890410958903 0.9816494845360824 0.026894506425645916\n",
+      "The chi-square score for 'it is' is : 33.61813303205762\n",
+      "Observed :  3 8 3 286\n",
+      "Expected :  0.22 10.78 5.78 283.22\n",
+      "X^2      :  35.129090909090905 0.716920222634508 1.3370934256055365 0.027287620930724694\n",
+      "The chi-square score for 'as a' is : 37.21039217826167\n"
+     ]
+    }
+   ],
+   "source": [
+    "for i in top_3:\n",
+    "    wrd1 = i.split(' ')[0]\n",
+    "    wrd2 = i.split(' ')[1]\n",
+    "    c_wrd1 = d[wrd1]\n",
+    "    c_n_wrd1 = nws - d[wrd1]\n",
+    "    c_wrd2 = d[wrd2]\n",
+    "    c_n_wrd2 = nws - d[wrd2]\n",
+    "    o_w1w2 = jd[i]\n",
+    "    o_nw1w2 = c_wrd2 - o_w1w2\n",
+    "    o_w1nw2 = c_wrd1 - o_w1w2\n",
+    "    o_nw1nw2 = c_n_wrd1 - o_nw1w2\n",
+    "    e_w1w2 = float((c_wrd1*c_wrd2)/nws)\n",
+    "    e_nw1w2 = float((c_n_wrd1*c_wrd2)/nws)\n",
+    "    e_w1nw2 = float((c_n_wrd2*c_wrd1)/nws)\n",
+    "    e_nw1nw2 = float((c_n_wrd2*c_n_wrd1)/nws)\n",
+    "    x_w1w2 = float(((o_w1w2 - e_w1w2)**2)/e_w1w2)\n",
+    "    x_nw1w2 = float(((o_nw1w2 - e_nw1w2)**2)/e_nw1w2)\n",
+    "    x_w1nw2 = float(((o_w1nw2 - e_w1nw2)**2)/e_w1nw2)\n",
+    "    x_nw1nw2 = float(((o_nw1nw2 - e_nw1nw2)**2)/e_nw1nw2)\n",
+    "    print(\"Observed : \",o_w1w2,o_nw1w2,o_w1nw2,o_nw1nw2)\n",
+    "    #print(\"Observed Total : \",o_w1w2+o_nw1w2+o_w1nw2+o_nw1nw2)\n",
+    "    print(\"Expected : \",e_w1w2,e_nw1w2,e_w1nw2,e_nw1nw2)\n",
+    "    #print(\"Expected Total : \",e_w1w2+e_nw1w2+e_w1nw2+e_nw1nw2)\n",
+    "    chi_sq = float(x_w1w2 + x_nw1w2 + x_w1nw2 + x_nw1nw2)\n",
+    "    print(\"X^2      : \",x_w1w2 , x_nw1w2 , x_w1nw2 , x_nw1nw2)\n",
+    "    print(\"The chi-square score for '\"+i+\"' is : \"+str(chi_sq))"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}