noshot 0.1.0__py3-none-any.whl

noshot/data/AIDS CN NLP/NLP/NLP 1/Word2Vec.ipynb

@@ -0,0 +1,93 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0aZKJeRc2JQz"
+      },
+      "source": []
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 2,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "G-jPBBIk1fPh",
+        "outputId": "1504bba6-85fd-4a1a-c051-293ce006813a"
+      },
+      "outputs": [
+        {
+          "name": "stderr",
+          "output_type": "stream",
+          "text": [
+            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
+            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
+          ]
+        },
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Words most similar to 'programming':\n",
+            "the: 0.1783\n",
+            "i: 0.1607\n",
+            "tool: 0.1056\n",
+            "great: 0.0922\n",
+            "python: 0.0270\n"
+          ]
+        }
+      ],
+      "source": [
+        "#Implement word2vec model to explore the semantic similarity between the words.\n",
+        "import gensim\n",
+        "from gensim.models import Word2Vec\n",
+        "import nltk\n",
+        "nltk.download('punkt')\n",
+        "from nltk.tokenize import word_tokenize\n",
+        "sentences = ['I love programming', 'Python is great', 'I enjoy machine learning',\n",
+        "             'TensorFlow is a powerful tool', 'AI is the future']\n",
+        "\n",
+        "tokenized_sentences = [word_tokenize(sentence.lower()) for sentence in sentences]\n",
+        "\n",
+        "\n",
+        "model_w2v = Word2Vec(sentences=tokenized_sentences, vector_size=100, window=5, min_count=1, workers=4)\n",
+        "\n",
+        "#  word vectors (semantic similarity)\n",
+        "word = 'programming'\n",
+        "similar_words = model_w2v.wv.most_similar(word, topn=5)\n",
+        "print(f\"Words most similar to '{word}':\")\n",
+        "for sim_word, sim_score in similar_words:\n",
+        "    print(f\"{sim_word}: {sim_score:.4f}\")\n",
+        "model_w2v.save(\"word2vec_model.bin\")\n",
+        "# loaded_model = Word2Vec.load(\"word2vec_model.bin\")\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ZjkrZu5411-K"
+      },
+      "outputs": [],
+      "source": []
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "gpuType": "T4",
+      "provenance": []
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "name": "python"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

noshot/data/AIDS CN NLP/NLP/NLP 2/Lab01(tokenizer)/tokenizer.ipynb

@@ -0,0 +1,370 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "e764270d",
+   "metadata": {},
+   "source": [
+    "Credits - **Geeks for Geeks**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "247f32b9",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "hello\n"
+     ]
+    }
+   ],
+   "source": [
+    "print('hello')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "33e4bd2e",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'run'"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import nltk\n",
+    "from nltk.stem import PorterStemmer\n",
+    "word_stemmer = PorterStemmer()\n",
+    "word_stemmer.stem('running')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "680512a3",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['Any',\n",
+       " 'sentence',\n",
+       " 'in',\n",
+       " 'the',\n",
+       " 'world',\n",
+       " 'ca',\n",
+       " \"n't\",\n",
+       " 'end',\n",
+       " 'with',\n",
+       " 'because',\n",
+       " 'because',\n",
+       " 'because',\n",
+       " 'is',\n",
+       " 'an',\n",
+       " 'conjunction',\n",
+       " '.']"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import nltk\n",
+    "from nltk.tokenize import word_tokenize\n",
+    "word_tokenize(\"Any sentence in the world can't end with because because because is an conjunction.\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "6cf8dab3",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['Any',\n",
+       " 'sentence',\n",
+       " 'in',\n",
+       " 'the',\n",
+       " 'world',\n",
+       " 'ca',\n",
+       " \"n't\",\n",
+       " 'end',\n",
+       " 'with',\n",
+       " 'because',\n",
+       " 'because',\n",
+       " 'because',\n",
+       " 'is',\n",
+       " 'an',\n",
+       " 'conjunction',\n",
+       " '.']"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import nltk\n",
+    "from nltk.tokenize import TreebankWordTokenizer\n",
+    "Tokenizer_wrd = TreebankWordTokenizer()\n",
+    "Tokenizer_wrd.tokenize(\n",
+    "   \"Any sentence in the world can't end with because because because is an conjunction.\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "b82ec031",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['won', '’', 't']"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import nltk\n",
+    "from nltk.tokenize import word_tokenize\n",
+    "word_tokenize('won’t')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "3f19ab5e",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['I', 'can', \"'\", 't', 'allow', 'you', 'to', 'go', 'home', 'early']"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from nltk.tokenize import WordPunctTokenizer\n",
+    "tokenizer = WordPunctTokenizer()\n",
+    "tokenizer.tokenize(\" I can't allow you to go home early\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "d24a533b",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['Let us understand the difference between sentence & word tokenizer.',\n",
+       " 'It is going to be a simple example.']"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import nltk\n",
+    "from nltk.tokenize import sent_tokenize\n",
+    "text = '''Let us understand the difference between sentence & word tokenizer. \n",
+    "It is going to be a simple example.'''\n",
+    "sent_tokenize(text)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "ac9fa608",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[\"won't is a contraction.\"]"
+      ]
+     },
+     "execution_count": 16,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import nltk\n",
+    "from nltk.tokenize import RegexpTokenizer\n",
+    "tokenizer = RegexpTokenizer('/s+' , gaps = True)\n",
+    "tokenizer.tokenize(\"won't is a contraction.\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "id": "e82b05b3",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Guy: How old are you?\n"
+     ]
+    }
+   ],
+   "source": [
+    "from nltk.tokenize import PunktSentenceTokenizer\n",
+    "from nltk.corpus import webtext\n",
+    "text = webtext.raw('C://Users/admin/Downloads/Lab1/training_tokenizer.txt')\n",
+    "sent_tokenizer = PunktSentenceTokenizer(text)\n",
+    "sents_1 = sent_tokenizer.tokenize(text)\n",
+    "print(sents_1[0])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "id": "cebb3ff8",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Guy: How old are you?\n"
+     ]
+    }
+   ],
+   "source": [
+    "from nltk.tokenize import PunktSentenceTokenizer\n",
+    "from nltk.corpus import webtext\n",
+    "text = webtext.raw('C://Users/admin/Downloads/Lab1/training_tokenizer.txt')\n",
+    "sent_tokenizer = PunktSentenceTokenizer(text)\n",
+    "sents_1 = sent_tokenizer.tokenize(text)\n",
+    "print(sents_1[0])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "id": "58aebec1",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['I', 'writer']"
+      ]
+     },
+     "execution_count": 25,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from nltk.corpus import stopwords\n",
+    "english_stops = set(stopwords.words('english'))\n",
+    "words = ['I', 'am', 'a', 'writer']\n",
+    "[word for word in words if word not in english_stops]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "id": "6ce95d73",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['arabic',\n",
+       " 'azerbaijani',\n",
+       " 'basque',\n",
+       " 'bengali',\n",
+       " 'catalan',\n",
+       " 'chinese',\n",
+       " 'danish',\n",
+       " 'dutch',\n",
+       " 'english',\n",
+       " 'finnish',\n",
+       " 'french',\n",
+       " 'german',\n",
+       " 'greek',\n",
+       " 'hebrew',\n",
+       " 'hinglish',\n",
+       " 'hungarian',\n",
+       " 'indonesian',\n",
+       " 'italian',\n",
+       " 'kazakh',\n",
+       " 'nepali',\n",
+       " 'norwegian',\n",
+       " 'portuguese',\n",
+       " 'romanian',\n",
+       " 'russian',\n",
+       " 'slovene',\n",
+       " 'spanish',\n",
+       " 'swedish',\n",
+       " 'tajik',\n",
+       " 'turkish']"
+      ]
+     },
+     "execution_count": 26,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from nltk.corpus import stopwords\n",
+    "stopwords.fileids()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

noshot/data/AIDS CN NLP/NLP/NLP 2/Lab01(tokenizer)/training_tokenizer.txt

@@ -0,0 +1,6 @@
+Guy: How old are you?
+Hipster girl: You know, I never answer that question. Because to me, it's about
+how mature you are, you know? I mean, a fourteen year old could be more mature
+than a twenty-five year old, right? I'm sorry, I just never answer that question.
+Guy: But, uh, you're older than eighteen, right?
+Hipster girl: Oh, yeah.