noshot 0.1.0__py3-none-any.whl

noshot/data/AIDS CN NLP/NLP/NLP 1/WSD.ipynb

@@ -0,0 +1,645 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 50,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "from nltk.util import bigrams\n",
+    "from nltk.corpus import stopwords\n",
+    "from nltk.tokenize import word_tokenize\n",
+    "import string"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 51,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Sentence</th>\n",
+       "      <th>Class</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>I deposited my paycheck at the bank.</td>\n",
+       "      <td>Financial Institution</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>The bank of the river was eroded by the heavy ...</td>\n",
+       "      <td>River Border</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>She works at the bank as a teller.</td>\n",
+       "      <td>Financial Institution</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>Let's have a picnic by the bank of the river.</td>\n",
+       "      <td>River Border</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>The bank approved my loan application.</td>\n",
+       "      <td>Financial Institution</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                            Sentence                  Class\n",
+       "0               I deposited my paycheck at the bank.  Financial Institution\n",
+       "1  The bank of the river was eroded by the heavy ...           River Border\n",
+       "2                She works at the bank as a teller.   Financial Institution\n",
+       "3     Let's have a picnic by the bank of the river.            River Border\n",
+       "4            The bank approved my loan application.   Financial Institution"
+      ]
+     },
+     "execution_count": 51,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df = pd.read_csv(\"../Bank.csv\")\n",
+    "df = df.iloc[:94]\n",
+    "df.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 52,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "stops = set(stopwords.words(\"english\"))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 53,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Sentence</th>\n",
+       "      <th>Class</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>[deposited, paycheck, bank]</td>\n",
+       "      <td>Financial Institution</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>[bank, river, eroded, heavy, rains]</td>\n",
+       "      <td>River Border</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>[works, bank, teller]</td>\n",
+       "      <td>Financial Institution</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>[let, 's, picnic, bank, river]</td>\n",
+       "      <td>River Border</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>[bank, approved, loan, application]</td>\n",
+       "      <td>Financial Institution</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                              Sentence                  Class\n",
+       "0          [deposited, paycheck, bank]  Financial Institution\n",
+       "1  [bank, river, eroded, heavy, rains]           River Border\n",
+       "2                [works, bank, teller]  Financial Institution\n",
+       "3       [let, 's, picnic, bank, river]           River Border\n",
+       "4  [bank, approved, loan, application]  Financial Institution"
+      ]
+     },
+     "execution_count": 53,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "for _,row in df.iterrows():\n",
+    "    row[\"Sentence\"] = row[\"Sentence\"].lower()\n",
+    "    row[\"Sentence\"] = word_tokenize(row[\"Sentence\"])\n",
+    "    row[\"Sentence\"] = [i for i in row[\"Sentence\"] if i not in stops and i not in string.punctuation]\n",
+    "df.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 54,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[['deposited', 'paycheck', 'bank'],\n",
+       " ['bank', 'river', 'eroded', 'heavy', 'rains'],\n",
+       " ['works', 'bank', 'teller'],\n",
+       " ['let', \"'s\", 'picnic', 'bank', 'river'],\n",
+       " ['bank', 'approved', 'loan', 'application']]"
+      ]
+     },
+     "execution_count": 54,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "sentences = df[\"Sentence\"]\n",
+    "sentences = sentences.tolist()\n",
+    "sentences[:5]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 55,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['Financial Institution',\n",
+       " 'River Border',\n",
+       " 'Financial Institution',\n",
+       " 'River Border',\n",
+       " 'Financial Institution']"
+      ]
+     },
+     "execution_count": 55,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "classes = df[\"Class\"]\n",
+    "classes = classes.tolist()\n",
+    "classes[:5]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Class Frequency Counters"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 56,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from collections import defaultdict"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 57,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "F_freq = defaultdict(int)\n",
+    "R_freq = defaultdict(int)\n",
+    "length = 0\n",
+    "F_counter = 0\n",
+    "R_counter = 0"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 58,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(45, 48)"
+      ]
+     },
+     "execution_count": 58,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "for i in range(94):\n",
+    "    length = length + 1\n",
+    "    if classes[i] == \"Financial Institution\":\n",
+    "        F_counter += 1\n",
+    "\n",
+    "        for j in sentences[i]:\n",
+    "            F_freq[j] += 1\n",
+    "    if classes[i] == \"River Border\":\n",
+    "        R_counter += 1\n",
+    "\n",
+    "        for j in sentences[i]:\n",
+    "            F_freq[j] += 1     \n",
+    "F_counter, R_counter"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Priors"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 59,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import math"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 60,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(-1.0473057147783567, -0.9541963103868752)"
+      ]
+     },
+     "execution_count": 60,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "total_classes = F_counter + R_counter\n",
+    "prior_fin = math.log2(F_counter/total_classes)\n",
+    "prior_riv = math.log2(R_counter/total_classes)\n",
+    "\n",
+    "prior_fin, prior_riv"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 61,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "255"
+      ]
+     },
+     "execution_count": 61,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "Vocab = set(list(F_freq.keys()) + list(R_freq.keys()))\n",
+    "V = len(Vocab)\n",
+    "V"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Test"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 64,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Sentence</th>\n",
+       "      <th>Class</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>95</th>\n",
+       "      <td>I need to update my contact information with t...</td>\n",
+       "      <td>?</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>96</th>\n",
+       "      <td>The bank provides online banking services for ...</td>\n",
+       "      <td>?</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>97</th>\n",
+       "      <td>The beavers constructed a dam along the bank o...</td>\n",
+       "      <td>?</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>98</th>\n",
+       "      <td>I need to check my transaction history at the ...</td>\n",
+       "      <td>?</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>99</th>\n",
+       "      <td>She works as a financial consultant at the bank.</td>\n",
+       "      <td>?</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                             Sentence Class\n",
+       "95  I need to update my contact information with t...     ?\n",
+       "96  The bank provides online banking services for ...     ?\n",
+       "97  The beavers constructed a dam along the bank o...     ?\n",
+       "98  I need to check my transaction history at the ...     ?\n",
+       "99  She works as a financial consultant at the bank.      ?"
+      ]
+     },
+     "execution_count": 64,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df = pd.read_csv(\"../Bank.csv\")\n",
+    "test = df.iloc[95:, :]\n",
+    "test.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 67,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Sentence</th>\n",
+       "      <th>Class</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>95</th>\n",
+       "      <td>[need, update, contact, information, bank]</td>\n",
+       "      <td>?</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>96</th>\n",
+       "      <td>[bank, provides, online, banking, services, co...</td>\n",
+       "      <td>?</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>97</th>\n",
+       "      <td>[beavers, constructed, dam, along, bank, river]</td>\n",
+       "      <td>?</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>98</th>\n",
+       "      <td>[need, check, transaction, history, bank]</td>\n",
+       "      <td>?</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>99</th>\n",
+       "      <td>[works, financial, consultant, bank]</td>\n",
+       "      <td>?</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                             Sentence Class\n",
+       "95         [need, update, contact, information, bank]     ?\n",
+       "96  [bank, provides, online, banking, services, co...     ?\n",
+       "97    [beavers, constructed, dam, along, bank, river]     ?\n",
+       "98          [need, check, transaction, history, bank]     ?\n",
+       "99               [works, financial, consultant, bank]     ?"
+      ]
+     },
+     "execution_count": 67,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "for _,row in test.iterrows():\n",
+    "    row[\"Sentence\"] = row[\"Sentence\"].lower()\n",
+    "    row[\"Sentence\"] = word_tokenize(row[\"Sentence\"])\n",
+    "    row[\"Sentence\"] = [i for i in row[\"Sentence\"] if i not in stops and i not in string.punctuation]\n",
+    "test.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 78,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Sentence</th>\n",
+       "      <th>Class</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>95</th>\n",
+       "      <td>[need, update, contact, information, bank]</td>\n",
+       "      <td>Financial Institution</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>96</th>\n",
+       "      <td>[bank, provides, online, banking, services, co...</td>\n",
+       "      <td>Financial Institution</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>97</th>\n",
+       "      <td>[beavers, constructed, dam, along, bank, river]</td>\n",
+       "      <td>Financial Institution</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>98</th>\n",
+       "      <td>[need, check, transaction, history, bank]</td>\n",
+       "      <td>Financial Institution</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>99</th>\n",
+       "      <td>[works, financial, consultant, bank]</td>\n",
+       "      <td>Financial Institution</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                             Sentence                  Class\n",
+       "95         [need, update, contact, information, bank]  Financial Institution\n",
+       "96  [bank, provides, online, banking, services, co...  Financial Institution\n",
+       "97    [beavers, constructed, dam, along, bank, river]  Financial Institution\n",
+       "98          [need, check, transaction, history, bank]  Financial Institution\n",
+       "99               [works, financial, consultant, bank]  Financial Institution"
+      ]
+     },
+     "execution_count": 78,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "for _, row in test.iterrows():\n",
+    "    score_fin = prior_fin\n",
+    "    score_riv = prior_riv\n",
+    "\n",
+    "    for token in row[\"Sentence\"]:\n",
+    "        score_fin += math.log2(F_freq[token] + 1) - math.log2(F_counter + V)\n",
+    "        score_riv += math.log2(R_freq[token] + 1) - math.log2(R_counter + V)\n",
+    "\n",
+    "    if score_fin > score_riv:\n",
+    "        row[\"Class\"] = \"Financial Institution\"\n",
+    "    else:\n",
+    "        row[\"Class\"] = \"River Border\"\n",
+    "\n",
+    "test"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}