noshot 0.1.0__py3-none-any.whl

noshot/data/AIDS CN NLP/NLP/NLP 3/ex-2-nlp.ipynb

@@ -0,0 +1,267 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "4f37e273-c9bd-4447-b164-ae3d72d60c3b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "|   Rank | Bigram                      |   Frequency |   Mean Probability (μ-value) |\n",
+      "|--------+-----------------------------+-------------+------------------------------|\n",
+      "|      1 | ('data', 'science')         |          15 |                     0.020243 |\n",
+      "|      2 | ('data', 'processing')      |           7 |                     0.009447 |\n",
+      "|      3 | ('predictive', 'analytics') |           5 |                     0.006748 |\n",
+      "|      4 | ('data', 'visualization')   |           5 |                     0.006748 |\n",
+      "|      5 | ('ai', 'data')              |           4 |                     0.005398 |\n",
+      "|      6 | ('ai', 'algorithms')        |           4 |                     0.005398 |\n",
+      "|      7 | ('data', 'cleaning')        |           4 |                     0.005398 |\n",
+      "|      8 | ('cleaning', 'preparation') |           4 |                     0.005398 |\n",
+      "|      9 | ('natural', 'language')     |           4 |                     0.005398 |\n",
+      "|     10 | ('language', 'processing')  |           4 |                     0.005398 |\n"
+     ]
+    }
+   ],
+   "source": [
+    "import nltk\n",
+    "from nltk.tokenize import word_tokenize\n",
+    "from nltk.corpus import stopwords\n",
+    "from nltk.probability import FreqDist\n",
+    "from tabulate import tabulate\n",
+    "\n",
+    "def preprocess_text(text: str) -> list[str]:\n",
+    "    \"\"\"Preprocesses the text by tokenizing, removing punctuation and stopwords.\"\"\"\n",
+    "    words = word_tokenize(text)\n",
+    "    stop_words = set(stopwords.words('english'))\n",
+    "    return [word.lower() for word in words if word.isalnum() and word.lower() not in stop_words]\n",
+    "\n",
+    "def calculate_mean_probability(bigram_freq: FreqDist, total_bigrams: int) -> dict:\n",
+    "    \"\"\"Calculates the mean probability (μ-value) of each bigram.\"\"\"\n",
+    "    mean_probabilities = {bigram: freq / total_bigrams for bigram, freq in bigram_freq.items()}\n",
+    "    return mean_probabilities\n",
+    "\n",
+    "def main():\n",
+    "    # Download required NLTK data\n",
+    "    # nltk.download('punkt')\n",
+    "    # nltk.download('stopwords')\n",
+    "\n",
+    "    # Load text\n",
+    "    with open('text3.txt', 'r') as file:\n",
+    "        text = file.read()\n",
+    "\n",
+    "    # Preprocess text\n",
+    "    words = preprocess_text(text)\n",
+    "\n",
+    "    # Calculate word frequency distribution\n",
+    "    fdist = FreqDist(words)\n",
+    "\n",
+    "    # Calculate bigrams and their frequencies\n",
+    "    bigrams = list(nltk.bigrams(words))\n",
+    "    bigram_freq = FreqDist(bigrams)\n",
+    "\n",
+    "    # Calculate mean probability (μ-value) for each bigram\n",
+    "    mean_probabilities = calculate_mean_probability(bigram_freq, len(bigrams))\n",
+    "\n",
+    "    # Sort collocations by mean probability\n",
+    "    collocations = sorted(mean_probabilities.items(), key=lambda x: x[1], reverse=True)\n",
+    "\n",
+    "    '''# Calculate mean of bigram frequencies\n",
+    "    total_bigram_freq = sum(bigram_freq.values())\n",
+    "    mean_bigram_freq = total_bigram_freq / len(bigram_freq)\n",
+    "\n",
+    "    # Print mean bigram frequency\n",
+    "    print(f\"Mean Bigram Frequency: {mean_bigram_freq:.2f}\\n\")'''\n",
+    "\n",
+    "    # Print top N collocations with their frequencies and mean probabilities\n",
+    "    N = 10\n",
+    "    headers = [\"Rank\", \"Bigram\", \"Frequency\", \"Mean Probability (μ-value)\"]\n",
+    "    table = []\n",
+    "    for i, (bigram, mean_prob) in enumerate(collocations[:N]):\n",
+    "        table.append([i+1, bigram, bigram_freq[bigram], f\"{mean_prob:.6f}\"])\n",
+    "    print(tabulate(table, headers, tablefmt=\"orgtbl\"))\n",
+    "\n",
+    "if __name__ == \"__main__\":\n",
+    "    main()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "1ba8ff30-565b-471a-a161-e47e0eb72cbc",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Mean Bigram Frequency: 1.17\n",
+      "\n",
+      "+--------+--------------------------------+-------------+----------------+---------------+-------------------+--------------+-----------------------+\n",
+      "|   Rank | Bigram                         |   Frequency |   Mean Prob(μ) |   t-Statistic |   p-Value(t-Test) |   Chi Square |   p-Value(Chi-Square) |\n",
+      "+========+================================+=============+================+===============+===================+==============+=======================+\n",
+      "|      1 | ('impact', 'artificial')       |           2 |       0.002699 |        1.4152 |            0.1574 |     183.747  |                0      |\n",
+      "+--------+--------------------------------+-------------+----------------+---------------+-------------------+--------------+-----------------------+\n",
+      "|      2 | ('artificial', 'intelligence') |           3 |       0.004049 |        1.7344 |            0.0833 |     307.077  |                0      |\n",
+      "+--------+--------------------------------+-------------+----------------+---------------+-------------------+--------------+-----------------------+\n",
+      "|      3 | ('intelligence', 'data')       |           3 |       0.004049 |        1.7344 |            0.0833 |      12.4097 |                0.0004 |\n",
+      "+--------+--------------------------------+-------------+----------------+---------------+-------------------+--------------+-----------------------+\n",
+      "|      4 | ('data', 'science')            |          15 |       0.020243 |        3.9101 |            0.0001 |     167.483  |                0      |\n",
+      "+--------+--------------------------------+-------------+----------------+---------------+-------------------+--------------+-----------------------+\n",
+      "|      5 | ('science', 'introduction')    |           1 |       0.00135  |        1      |            0.3176 |      11.6208 |                0.0007 |\n",
+      "+--------+--------------------------------+-------------+----------------+---------------+-------------------+--------------+-----------------------+\n",
+      "|      6 | ('introduction', 'recent')     |           1 |       0.00135  |        1      |            0.3176 |     184.75   |                0      |\n",
+      "+--------+--------------------------------+-------------+----------------+---------------+-------------------+--------------+-----------------------+\n",
+      "|      7 | ('recent', 'years')            |           1 |       0.00135  |        1      |            0.3176 |     184.75   |                0      |\n",
+      "+--------+--------------------------------+-------------+----------------+---------------+-------------------+--------------+-----------------------+\n",
+      "|      8 | ('years', 'convergence')       |           1 |       0.00135  |        1      |            0.3176 |     184.75   |                0      |\n",
+      "+--------+--------------------------------+-------------+----------------+---------------+-------------------+--------------+-----------------------+\n",
+      "|      9 | ('convergence', 'artificial')  |           1 |       0.00135  |        1      |            0.3176 |      61.0835 |                0      |\n",
+      "+--------+--------------------------------+-------------+----------------+---------------+-------------------+--------------+-----------------------+\n",
+      "|     10 | ('intelligence', 'ai')         |           1 |       0.00135  |        1      |            0.3176 |       0.4959 |                0.4813 |\n",
+      "+--------+--------------------------------+-------------+----------------+---------------+-------------------+--------------+-----------------------+\n"
+     ]
+    }
+   ],
+   "source": [
+    "import nltk\n",
+    "from nltk.tokenize import word_tokenize\n",
+    "from nltk.corpus import stopwords\n",
+    "from nltk.probability import FreqDist\n",
+    "from tabulate import tabulate\n",
+    "from scipy.stats import chi2_contingency, ttest_1samp\n",
+    "import numpy as np\n",
+    "\n",
+    "def preprocess_text(text: str) -> list[str]:\n",
+    "    \"\"\"Preprocesses the text by tokenizing, removing punctuation and stopwords.\"\"\"\n",
+    "    words = word_tokenize(text)\n",
+    "    stop_words = set(stopwords.words('english'))\n",
+    "    return [word.lower() for word in words if word.isalnum() and word.lower() not in stop_words]\n",
+    "\n",
+    "def calculate_mean_probability(bigram_freq: FreqDist, total_bigrams: int) -> dict:\n",
+    "    \"\"\"Calculates the mean probability (μ-value) of each bigram.\"\"\"\n",
+    "    mean_probabilities = {bigram: freq / total_bigrams for bigram, freq in bigram_freq.items()}\n",
+    "    return mean_probabilities\n",
+    "\n",
+    "def perform_statistical_tests(bigram_freq: FreqDist, word_freq: FreqDist, total_bigrams: int):\n",
+    "    \"\"\"Perform t-test and chi-square test for each bigram.\"\"\"\n",
+    "    results = []\n",
+    "\n",
+    "    for bigram, observed_freq in bigram_freq.items():\n",
+    "        word1, word2 = bigram\n",
+    "        freq_w1 = word_freq.get(word1, 0)\n",
+    "        freq_w2 = word_freq.get(word2, 0)\n",
+    "        \n",
+    "        # Expected frequency for the bigram assuming independence\n",
+    "        expected_freq = (freq_w1 * freq_w2) / total_bigrams\n",
+    "        \n",
+    "        # Chi-square test\n",
+    "        observed = np.array([\n",
+    "            [observed_freq, freq_w1 - observed_freq],\n",
+    "            [freq_w2 - observed_freq, total_bigrams - (freq_w1 + freq_w2 - observed_freq)]\n",
+    "        ])\n",
+    "        \n",
+    "        try:\n",
+    "            chi2_stat, p_value_chi2, dof, ex = chi2_contingency(observed)\n",
+    "        except ValueError:\n",
+    "            chi2_stat, p_value_chi2 = np.nan, np.nan\n",
+    "        \n",
+    "        # Generate sample data to perform t-test\n",
+    "        sample_data = [observed_freq] * observed_freq + [expected_freq] * (total_bigrams - observed_freq)\n",
+    "        \n",
+    "        # Perform one-sample t-test\n",
+    "        t_stat, p_value_t = ttest_1samp(sample_data, expected_freq)\n",
+    "        \n",
+    "        results.append((bigram, observed_freq, t_stat, p_value_t, chi2_stat, p_value_chi2))\n",
+    "    \n",
+    "    return results\n",
+    "\n",
+    "def main():\n",
+    "    # Download required NLTK data\n",
+    "    # nltk.download('punkt')\n",
+    "    # nltk.download('stopwords')\n",
+    "\n",
+    "    # Load text\n",
+    "    with open(\"text3.txt\", 'r') as file:\n",
+    "        text = file.read()\n",
+    "\n",
+    "    # Preprocess text\n",
+    "    words = preprocess_text(text)\n",
+    "\n",
+    "    # Calculate word frequency distribution\n",
+    "    word_freq = FreqDist(words)\n",
+    "\n",
+    "    # Calculate bigrams and their frequencies\n",
+    "    bigrams = list(nltk.bigrams(words))\n",
+    "    bigram_freq = FreqDist(bigrams)\n",
+    "\n",
+    "    # Calculate mean probability (μ-value) for each bigram\n",
+    "    mean_probabilities = calculate_mean_probability(bigram_freq, len(bigrams))\n",
+    "\n",
+    "    # Sort collocations by mean probability\n",
+    "    collocations = sorted(mean_probabilities.items(), key=lambda x: x[1], reverse=True)\n",
+    "\n",
+    "    # Calculate mean of bigram frequencies\n",
+    "    total_bigram_freq = sum(bigram_freq.values())\n",
+    "    mean_bigram_freq = total_bigram_freq / len(bigram_freq)\n",
+    "\n",
+    "    # Print mean bigram frequency\n",
+    "    print(f\"Mean Bigram Frequency: {mean_bigram_freq:.2f}\\n\")\n",
+    "\n",
+    "    # Perform statistical tests for each bigram\n",
+    "    results = perform_statistical_tests(bigram_freq, word_freq, len(bigrams))\n",
+    "\n",
+    "    # Print top N collocations with their frequencies and mean probabilities\n",
+    "    N = 10\n",
+    "    headers = [\"Rank\", \"Bigram\", \"Frequency\", \"Mean Prob(μ)\", \"t-Statistic\", \"p-Value(t-Test)\", \"Chi Square\", \"p-Value(Chi-Square)\"]\n",
+    "    table = []\n",
+    "    for i, (bigram, observed_freq, t_stat, p_value_t, chi2_stat, p_value_chi2) in enumerate(results[:N]):\n",
+    "        table.append([\n",
+    "            i + 1,\n",
+    "            bigram,\n",
+    "            observed_freq,\n",
+    "            f\"{mean_probabilities.get(bigram, 0):.6f}\",\n",
+    "            f\"{t_stat:.4f}\" if not np.isnan(t_stat) else \"NaN\",\n",
+    "            f\"{p_value_t:.4f}\" if not np.isnan(p_value_t) else \"NaN\",\n",
+    "            f\"{chi2_stat:.4f}\" if not np.isnan(chi2_stat) else \"NaN\",\n",
+    "            f\"{p_value_chi2:.4f}\" if not np.isnan(p_value_chi2) else \"NaN\"\n",
+    "        ])\n",
+    "    print(tabulate(table, headers, tablefmt=\"grid\"))\n",
+    "\n",
+    "if __name__ == \"__main__\":\n",
+    "    main()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "35ee8280-0449-4fb4-846e-3128f29a09cb",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

noshot/data/AIDS CN NLP/NLP/NLP 3/exp8&9.ipynb

@@ -0,0 +1,305 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "b644056b-f928-4b45-9229-7e487a552005",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "            one     child        ''      time      like   without     littl  \\\n",
+      "one    1.000000 -0.005589 -0.036006 -0.100524 -0.021266 -0.039723  0.029068   \n",
+      "child -0.005589  1.000000 -0.019693  0.067477  0.002063  0.015178 -0.108766   \n",
+      "''    -0.036006 -0.019693  1.000000 -0.003567  0.172956  0.083302  0.159254   \n",
+      "time  -0.100524  0.067477 -0.003567  1.000000 -0.048268  0.142675  0.052523   \n",
+      "like  -0.021266  0.002063  0.172956 -0.048268  1.000000  0.145309  0.040206   \n",
+      "\n",
+      "           life      word     could  ...    vitiat     brain      real  \\\n",
+      "one    0.107384  0.035972  0.222857  ... -0.022165  0.125102  0.026518   \n",
+      "child -0.108151  0.034731 -0.094406  ... -0.046126 -0.048296 -0.017403   \n",
+      "''     0.014497  0.207438 -0.025448  ... -0.067862 -0.069103 -0.085746   \n",
+      "time  -0.003863  0.078271 -0.163900  ... -0.103645  0.090741 -0.041755   \n",
+      "like  -0.023750 -0.067008 -0.174552  ... -0.180237 -0.130274 -0.028213   \n",
+      "\n",
+      "         physic     moral    situat      draw      back  insurmount  \\\n",
+      "one    0.153639 -0.013773 -0.083296 -0.011102 -0.261834   -0.143206   \n",
+      "child  0.021467 -0.087260  0.167192 -0.035414  0.061198   -0.088217   \n",
+      "''     0.144862 -0.039488 -0.066403  0.200111 -0.104986   -0.135296   \n",
+      "time   0.122950 -0.169851  0.139709 -0.024709  0.100319    0.028166   \n",
+      "like   0.085665 -0.126780  0.174523  0.037570 -0.040087    0.108647   \n",
+      "\n",
+      "       veziers-le-rethel  \n",
+      "one             0.031907  \n",
+      "child          -0.138646  \n",
+      "''             -0.073759  \n",
+      "time            0.016197  \n",
+      "like           -0.043849  \n",
+      "\n",
+      "[5 rows x 651 columns]\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "C:\\Users\\admin\\anaconda3\\Lib\\site-packages\\keras\\src\\layers\\core\\input_layer.py:25: UserWarning: Argument `input_shape` is deprecated. Use `shape` instead.\n",
+      "  warnings.warn(\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Model: \"sequential_2\"</span>\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "\u001b[1mModel: \"sequential_2\"\u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n",
+       "┃<span style=\"font-weight: bold\"> Layer (type)                    </span>┃<span style=\"font-weight: bold\"> Output Shape           </span>┃<span style=\"font-weight: bold\">       Param # </span>┃\n",
+       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n",
+       "│ dense_6 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)                 │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)             │         <span style=\"color: #00af00; text-decoration-color: #00af00\">6,464</span> │\n",
+       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
+       "│ dense_7 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)                 │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">32</span>)             │         <span style=\"color: #00af00; text-decoration-color: #00af00\">2,080</span> │\n",
+       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
+       "│ dense_8 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)                 │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1</span>)              │            <span style=\"color: #00af00; text-decoration-color: #00af00\">33</span> │\n",
+       "└─────────────────────────────────┴────────────────────────┴───────────────┘\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n",
+       "┃\u001b[1m \u001b[0m\u001b[1mLayer (type)                   \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape          \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m      Param #\u001b[0m\u001b[1m \u001b[0m┃\n",
+       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n",
+       "│ dense_6 (\u001b[38;5;33mDense\u001b[0m)                 │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m64\u001b[0m)             │         \u001b[38;5;34m6,464\u001b[0m │\n",
+       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
+       "│ dense_7 (\u001b[38;5;33mDense\u001b[0m)                 │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m32\u001b[0m)             │         \u001b[38;5;34m2,080\u001b[0m │\n",
+       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
+       "│ dense_8 (\u001b[38;5;33mDense\u001b[0m)                 │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m1\u001b[0m)              │            \u001b[38;5;34m33\u001b[0m │\n",
+       "└─────────────────────────────────┴────────────────────────┴───────────────┘\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Total params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">8,577</span> (33.50 KB)\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "\u001b[1m Total params: \u001b[0m\u001b[38;5;34m8,577\u001b[0m (33.50 KB)\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">8,577</span> (33.50 KB)\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m8,577\u001b[0m (33.50 KB)\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Non-trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (0.00 B)\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Epoch 1/10\n",
+      "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 2s/step - accuracy: 1.0000 - loss: 0.6929\n",
+      "Epoch 2/10\n",
+      "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 26ms/step - accuracy: 1.0000 - loss: 0.6909\n",
+      "Epoch 3/10\n",
+      "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 23ms/step - accuracy: 1.0000 - loss: 0.6883\n",
+      "Epoch 4/10\n",
+      "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 31ms/step - accuracy: 1.0000 - loss: 0.6856\n",
+      "Epoch 5/10\n",
+      "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 24ms/step - accuracy: 1.0000 - loss: 0.6830\n",
+      "Epoch 6/10\n",
+      "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 24ms/step - accuracy: 1.0000 - loss: 0.6804\n",
+      "Epoch 7/10\n",
+      "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 25ms/step - accuracy: 1.0000 - loss: 0.6776\n",
+      "Epoch 8/10\n",
+      "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 23ms/step - accuracy: 1.0000 - loss: 0.6748\n",
+      "Epoch 9/10\n",
+      "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 22ms/step - accuracy: 1.0000 - loss: 0.6719\n",
+      "Epoch 10/10\n",
+      "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 24ms/step - accuracy: 1.0000 - loss: 0.6690\n",
+      "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 146ms/step - accuracy: 1.0000 - loss: 0.6661\n",
+      "\n",
+      "Final loss: 0.6660556197166443\n",
+      "Final accuracy: 1.0\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Import required libraries\n",
+    "import nltk\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "import re\n",
+    "import seaborn as sns\n",
+    "import matplotlib.pyplot as plt\n",
+    "from gensim.models import Word2Vec\n",
+    "from nltk.corpus import stopwords\n",
+    "from nltk.tokenize import word_tokenize\n",
+    "from nltk.stem import WordNetLemmatizer, PorterStemmer\n",
+    "import tensorflow as tf\n",
+    "import string\n",
+    "\n",
+    "# Download required NLTK resources (uncomment if not already downloaded)\n",
+    "# nltk.download('punkt')\n",
+    "# nltk.download('stopwords')\n",
+    "# nltk.download('wordnet')\n",
+    "\n",
+    "# Initialize the stopwords, lemmatizer, and stemmer\n",
+    "stop_words = set(stopwords.words('english'))\n",
+    "lemmatizer = WordNetLemmatizer()\n",
+    "stemmer = PorterStemmer()\n",
+    "\n",
+    "# Function to preprocess text data (Tokenization, Lemmatization, Stemming, Stopwords Removal)\n",
+    "def preprocess_text(text):\n",
+    "    # Tokenize the text\n",
+    "    tokens = word_tokenize(text.lower())\n",
+    "    \n",
+    "    # Remove punctuation and stopwords, and perform lemmatization and stemming\n",
+    "    processed_tokens = []\n",
+    "    for word in tokens:\n",
+    "        if word not in stop_words and word not in string.punctuation:\n",
+    "            lemmatized_word = lemmatizer.lemmatize(word)  # Lemmatization\n",
+    "            stemmed_word = stemmer.stem(lemmatized_word)  # Stemming\n",
+    "            processed_tokens.append(stemmed_word)\n",
+    "    \n",
+    "    return processed_tokens\n",
+    "\n",
+    "# Read the text data file\n",
+    "file_path = 'text2.txt'  # You can change this to your actual file path\n",
+    "with open(file_path, 'r', encoding='utf-8') as file:\n",
+    "    text_data = file.read()\n",
+    "\n",
+    "# Preprocess the text data\n",
+    "tokens = preprocess_text(text_data)\n",
+    "\n",
+    "# ============================\n",
+    "# WORD2VEC PART\n",
+    "# ============================\n",
+    "\n",
+    "# Train Word2Vec model\n",
+    "word2vec_model = Word2Vec(sentences=[tokens], vector_size=100, window=5, min_count=1, sg=0)  # sg=0 for CBOW\n",
+    "\n",
+    "# Get the list of all unique words in the vocabulary\n",
+    "vocab = list(word2vec_model.wv.index_to_key)\n",
+    "\n",
+    "# Initialize a matrix to store similarity scores\n",
+    "similarity_matrix = np.zeros((len(vocab), len(vocab)))\n",
+    "\n",
+    "# Compute pairwise similarity for all words in the vocabulary\n",
+    "for i, word1 in enumerate(vocab):\n",
+    "    for j, word2 in enumerate(vocab):\n",
+    "        similarity_matrix[i, j] = word2vec_model.wv.similarity(word1, word2)\n",
+    "\n",
+    "# Convert similarity matrix into a pandas DataFrame for easy visualization\n",
+    "similarity_df = pd.DataFrame(similarity_matrix, index=vocab, columns=vocab)\n",
+    "\n",
+    "# Display the first few rows of the similarity matrix\n",
+    "print(similarity_df.head())\n",
+    "\n",
+    "# ============================\n",
+    "# NEURAL NETWORK PART\n",
+    "# ============================\n",
+    "\n",
+    "# Convert tokens to Word2Vec embeddings\n",
+    "word_embeddings = np.array([word2vec_model.wv[word] for word in tokens if word in word2vec_model.wv])\n",
+    "\n",
+    "# Aggregate Word2Vec embeddings by averaging\n",
+    "average_embedding = np.mean(word_embeddings, axis=0)\n",
+    "\n",
+    "# Reshape the averaged embedding to match the input expected by the model\n",
+    "average_embedding = average_embedding.reshape(1, -1)  # Reshape to (1, 100)\n",
+    "\n",
+    "# For demo purposes, we create a mock label (you can replace it with your real labels)\n",
+    "labels = np.array([1])  # Assuming binary classification (0 or 1), change based on your data\n",
+    "\n",
+    "# Define the ANN model\n",
+    "model = tf.keras.Sequential([\n",
+    "    tf.keras.layers.InputLayer(input_shape=(average_embedding.shape[1],)),  # Input layer based on averaged Word2Vec vector size\n",
+    "    tf.keras.layers.Dense(64, activation='relu'),  # First hidden layer with 64 neurons\n",
+    "    tf.keras.layers.Dense(32, activation='relu'),  # Second hidden layer with 32 neurons\n",
+    "    tf.keras.layers.Dense(1, activation='sigmoid')  # Output layer (binary classification)\n",
+    "])\n",
+    "\n",
+    "# Compile the model\n",
+    "model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n",
+    "\n",
+    "# Print model summary\n",
+    "model.summary()\n",
+    "\n",
+    "# Train the model on the averaged Word2Vec embedding\n",
+    "history = model.fit(average_embedding, labels, epochs=10, verbose=1)\n",
+    "\n",
+    "# Evaluate model performance\n",
+    "loss, accuracy = model.evaluate(average_embedding, labels, verbose=1)\n",
+    "print(f\"\\nFinal loss: {loss}\")\n",
+    "print(f\"Final accuracy: {accuracy}\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ec1ca09b-13eb-40a4-af30-3366f604ea29",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}