noshot 0.1.0__py3-none-any.whl

noshot/data/AIDS CN NLP/NLP/NLP 3/ex-1-nltk.ipynb

@@ -0,0 +1,711 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "62a71ce0-0a21-45ef-9204-16d27ff27b89",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Top 10 Collocations:\n",
+      "1. Bigram: ('accurate', 'actionable'), Frequency: 1\n",
+      "2. Bigram: ('accurately', 'model'), Frequency: 1\n",
+      "3. Bigram: ('action', 'supply'), Frequency: 1\n",
+      "4. Bigram: ('adjusting', 'parameters'), Frequency: 1\n",
+      "5. Bigram: ('advancements', 'technology'), Frequency: 1\n",
+      "6. Bigram: ('allow', 'users'), Frequency: 1\n",
+      "7. Bigram: ('anomalies', 'triggering'), Frequency: 1\n",
+      "8. Bigram: ('assess', 'credit'), Frequency: 1\n",
+      "9. Bigram: ('bandwidth', 'requirements'), Frequency: 1\n",
+      "10. Bigram: ('becomes', 'available'), Frequency: 1\n"
+     ]
+    }
+   ],
+   "source": [
+    "import nltk\n",
+    "from nltk.tokenize import word_tokenize\n",
+    "from nltk.corpus import stopwords\n",
+    "from nltk.probability import FreqDist\n",
+    "from nltk.collocations import BigramCollocationFinder\n",
+    "from nltk.metrics import BigramAssocMeasures\n",
+    "\n",
+    "# Download required NLTK data\n",
+    "#nltk.download('punkt')\n",
+    "#nltk.download('stopwords')\n",
+    "\n",
+    "# Load text\n",
+    "with open('text3.txt', 'r') as file:\n",
+    "    text = file.read()\n",
+    "\n",
+    "# Preprocess text\n",
+    "words = [word.lower() for word in word_tokenize(text) if word.isalnum() and word.lower() not in stopwords.words('english')]\n",
+    "\n",
+    "# Calculate bigrams and their frequencies\n",
+    "bigram_finder = BigramCollocationFinder.from_words(words)\n",
+    "bigram_freq = bigram_finder.ngram_fd\n",
+    "\n",
+    "# Calculate top collocations\n",
+    "bigram_measures = BigramAssocMeasures()\n",
+    "collocations = bigram_finder.nbest(bigram_measures.pmi, 10)\n",
+    "\n",
+    "# Print top collocations\n",
+    "print(\"Top 10 Collocations:\")\n",
+    "for i, collocation in enumerate(collocations):\n",
+    "    print(f\"{i+1}. Bigram: {collocation}, Frequency: {bigram_freq[collocation]}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "id": "d3ef1337-2135-48cd-b1e8-8aa0ab375450",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Mean Bigram Frequency: 1.17\n",
+      "\n",
+      "Top 10 Collocations by Mean Probability:\n",
+      "1. Bigram: ('data', 'science'),\t Frequency: 15, \tMean Probability (μ-value): 0.020243\n",
+      "2. Bigram: ('data', 'processing'),\t Frequency: 7, \tMean Probability (μ-value): 0.009447\n",
+      "3. Bigram: ('predictive', 'analytics'),\t Frequency: 5, \tMean Probability (μ-value): 0.006748\n",
+      "4. Bigram: ('data', 'visualization'),\t Frequency: 5, \tMean Probability (μ-value): 0.006748\n",
+      "5. Bigram: ('ai', 'data'),\t Frequency: 4, \tMean Probability (μ-value): 0.005398\n",
+      "6. Bigram: ('ai', 'algorithms'),\t Frequency: 4, \tMean Probability (μ-value): 0.005398\n",
+      "7. Bigram: ('data', 'cleaning'),\t Frequency: 4, \tMean Probability (μ-value): 0.005398\n",
+      "8. Bigram: ('cleaning', 'preparation'),\t Frequency: 4, \tMean Probability (μ-value): 0.005398\n",
+      "9. Bigram: ('natural', 'language'),\t Frequency: 4, \tMean Probability (μ-value): 0.005398\n",
+      "10. Bigram: ('language', 'processing'),\t Frequency: 4, \tMean Probability (μ-value): 0.005398\n"
+     ]
+    }
+   ],
+   "source": [
+    "import nltk\n",
+    "from nltk.tokenize import word_tokenize\n",
+    "from nltk.corpus import stopwords\n",
+    "from nltk.probability import FreqDist\n",
+    "\n",
+    "def preprocess_text(text: str) -> list[str]:\n",
+    "    \"\"\"Preprocesses the text by tokenizing, removing punctuation and stopwords.\"\"\"\n",
+    "    words = word_tokenize(text)\n",
+    "    stop_words = set(stopwords.words('english'))\n",
+    "    return [word.lower() for word in words if word.isalnum() and word.lower() not in stop_words]\n",
+    "\n",
+    "def calculate_mean_probability(bigram_freq: FreqDist, total_bigrams: int) -> dict:\n",
+    "    \"\"\"Calculates the mean probability (μ-value) of each bigram.\"\"\"\n",
+    "    mean_probabilities = {bigram: freq / total_bigrams for bigram, freq in bigram_freq.items()}\n",
+    "    return mean_probabilities\n",
+    "\n",
+    "def main():\n",
+    "    # Download required NLTK data\n",
+    "    # nltk.download('punkt')\n",
+    "    # nltk.download('stopwords')\n",
+    "\n",
+    "    # Load text\n",
+    "    with open('text3.txt', 'r') as file:\n",
+    "        text = file.read()\n",
+    "\n",
+    "    # Preprocess text\n",
+    "    words = preprocess_text(text)\n",
+    "\n",
+    "    # Calculate word frequency distribution\n",
+    "    fdist = FreqDist(words)\n",
+    "\n",
+    "    # Calculate bigrams and their frequencies\n",
+    "    bigrams = list(nltk.bigrams(words))\n",
+    "    bigram_freq = FreqDist(bigrams)\n",
+    "\n",
+    "    # Calculate mean probability (μ-value) for each bigram\n",
+    "    mean_probabilities = calculate_mean_probability(bigram_freq, len(bigrams))\n",
+    "\n",
+    "    # Sort collocations by mean probability\n",
+    "    collocations = sorted(mean_probabilities.items(), key=lambda x: x[1], reverse=True)\n",
+    "\n",
+    "    # Calculate mean of bigram frequencies\n",
+    "    total_bigram_freq = sum(bigram_freq.values())\n",
+    "    mean_bigram_freq = total_bigram_freq / len(bigram_freq)\n",
+    "\n",
+    "    # Print mean bigram frequency\n",
+    "    print(f\"Mean Bigram Frequency: {mean_bigram_freq:.2f}\\n\")\n",
+    "\n",
+    "    # Print top N collocations with their frequencies and mean probabilities\n",
+    "    N = 10\n",
+    "    print(\"Top\", N, \"Collocations by Mean Probability:\")\n",
+    "    for i, (bigram, mean_prob) in enumerate(collocations[:N]):\n",
+    "        print(f\"{i+1}. Bigram: {bigram},\\t Frequency: {bigram_freq[bigram]}, \\tMean Probability (μ-value): {mean_prob:.6f}\")\n",
+    "\n",
+    "if __name__ == \"__main__\":\n",
+    "    main()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "68a08d04-501c-425d-a497-c7fe1d9a0013",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "|   Rank | Bigram                      |   Frequency |   Mean Probability (μ-value) |\n",
+      "|--------+-----------------------------+-------------+------------------------------|\n",
+      "|      1 | ('data', 'science')         |          15 |                     0.020243 |\n",
+      "|      2 | ('data', 'processing')      |           7 |                     0.009447 |\n",
+      "|      3 | ('predictive', 'analytics') |           5 |                     0.006748 |\n",
+      "|      4 | ('data', 'visualization')   |           5 |                     0.006748 |\n",
+      "|      5 | ('ai', 'data')              |           4 |                     0.005398 |\n",
+      "|      6 | ('ai', 'algorithms')        |           4 |                     0.005398 |\n",
+      "|      7 | ('data', 'cleaning')        |           4 |                     0.005398 |\n",
+      "|      8 | ('cleaning', 'preparation') |           4 |                     0.005398 |\n",
+      "|      9 | ('natural', 'language')     |           4 |                     0.005398 |\n",
+      "|     10 | ('language', 'processing')  |           4 |                     0.005398 |\n"
+     ]
+    }
+   ],
+   "source": [
+    "import nltk\n",
+    "from nltk.tokenize import word_tokenize\n",
+    "from nltk.corpus import stopwords\n",
+    "from nltk.probability import FreqDist\n",
+    "from tabulate import tabulate\n",
+    "\n",
+    "def preprocess_text(text: str) -> list[str]:\n",
+    "    \"\"\"Preprocesses the text by tokenizing, removing punctuation and stopwords.\"\"\"\n",
+    "    words = word_tokenize(text)\n",
+    "    stop_words = set(stopwords.words('english'))\n",
+    "    return [word.lower() for word in words if word.isalnum() and word.lower() not in stop_words]\n",
+    "\n",
+    "def calculate_mean_probability(bigram_freq: FreqDist, total_bigrams: int) -> dict:\n",
+    "    \"\"\"Calculates the mean probability (μ-value) of each bigram.\"\"\"\n",
+    "    mean_probabilities = {bigram: freq / total_bigrams for bigram, freq in bigram_freq.items()}\n",
+    "    return mean_probabilities\n",
+    "\n",
+    "def main():\n",
+    "    # Download required NLTK data\n",
+    "    # nltk.download('punkt')\n",
+    "    # nltk.download('stopwords')\n",
+    "\n",
+    "    # Load text\n",
+    "    with open('text3.txt', 'r') as file:\n",
+    "        text = file.read()\n",
+    "\n",
+    "    # Preprocess text\n",
+    "    words = preprocess_text(text)\n",
+    "\n",
+    "    # Calculate word frequency distribution\n",
+    "    fdist = FreqDist(words)\n",
+    "\n",
+    "    # Calculate bigrams and their frequencies\n",
+    "    bigrams = list(nltk.bigrams(words))\n",
+    "    bigram_freq = FreqDist(bigrams)\n",
+    "\n",
+    "    # Calculate mean probability (μ-value) for each bigram\n",
+    "    mean_probabilities = calculate_mean_probability(bigram_freq, len(bigrams))\n",
+    "\n",
+    "    # Sort collocations by mean probability\n",
+    "    collocations = sorted(mean_probabilities.items(), key=lambda x: x[1], reverse=True)\n",
+    "\n",
+    "    '''# Calculate mean of bigram frequencies\n",
+    "    total_bigram_freq = sum(bigram_freq.values())\n",
+    "    mean_bigram_freq = total_bigram_freq / len(bigram_freq)\n",
+    "\n",
+    "    # Print mean bigram frequency\n",
+    "    print(f\"Mean Bigram Frequency: {mean_bigram_freq:.2f}\\n\")'''\n",
+    "\n",
+    "    # Print top N collocations with their frequencies and mean probabilities\n",
+    "    N = 10\n",
+    "    headers = [\"Rank\", \"Bigram\", \"Frequency\", \"Mean Probability (μ-value)\"]\n",
+    "    table = []\n",
+    "    for i, (bigram, mean_prob) in enumerate(collocations[:N]):\n",
+    "        table.append([i+1, bigram, bigram_freq[bigram], f\"{mean_prob:.6f}\"])\n",
+    "    print(tabulate(table, headers, tablefmt=\"orgtbl\"))\n",
+    "\n",
+    "if __name__ == \"__main__\":\n",
+    "    main()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "bdcf71a0-02f3-49ba-bf3b-09bb1c079fdf",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Mean Bigram Frequency: 1.17\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "C:\\Users\\admin\\anaconda3\\Lib\\site-packages\\scipy\\stats\\_stats_py.py:1103: RuntimeWarning: divide by zero encountered in divide\n",
+      "  var *= np.divide(n, n-ddof)  # to avoid error on division by zero\n",
+      "C:\\Users\\admin\\anaconda3\\Lib\\site-packages\\scipy\\stats\\_stats_py.py:1103: RuntimeWarning: invalid value encountered in scalar multiply\n",
+      "  var *= np.divide(n, n-ddof)  # to avoid error on division by zero\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "|   Rank | Bigram                         |   Frequency |   Mean Probability (μ-value) |   t-Statistic |   p-Value (t-Test) |   Chi2 Statistic |   p-Value (Chi-Square) |\n",
+      "|--------+--------------------------------+-------------+------------------------------+---------------+--------------------+------------------+------------------------|\n",
+      "|      1 | ('impact', 'artificial')       |           2 |                     0.002699 |           nan |                nan |         183.747  |                 0      |\n",
+      "|      2 | ('artificial', 'intelligence') |           3 |                     0.004049 |           nan |                nan |         307.077  |                 0      |\n",
+      "|      3 | ('intelligence', 'data')       |           3 |                     0.004049 |           nan |                nan |          12.4097 |                 0.0004 |\n",
+      "|      4 | ('data', 'science')            |          15 |                     0.020243 |           nan |                nan |         167.483  |                 0      |\n",
+      "|      5 | ('science', 'introduction')    |           1 |                     0.00135  |           nan |                nan |          11.6208 |                 0.0007 |\n",
+      "|      6 | ('introduction', 'recent')     |           1 |                     0.00135  |           nan |                nan |         184.75   |                 0      |\n",
+      "|      7 | ('recent', 'years')            |           1 |                     0.00135  |           nan |                nan |         184.75   |                 0      |\n",
+      "|      8 | ('years', 'convergence')       |           1 |                     0.00135  |           nan |                nan |         184.75   |                 0      |\n",
+      "|      9 | ('convergence', 'artificial')  |           1 |                     0.00135  |           nan |                nan |          61.0835 |                 0      |\n",
+      "|     10 | ('intelligence', 'ai')         |           1 |                     0.00135  |           nan |                nan |           0.4959 |                 0.4813 |\n"
+     ]
+    }
+   ],
+   "source": [
+    "import nltk\n",
+    "from nltk.tokenize import word_tokenize\n",
+    "from nltk.corpus import stopwords\n",
+    "from nltk.probability import FreqDist\n",
+    "from tabulate import tabulate\n",
+    "from scipy.stats import ttest_1samp, chi2_contingency\n",
+    "import numpy as np\n",
+    "\n",
+    "def preprocess_text(text: str) -> list[str]:\n",
+    "    \"\"\"Preprocesses the text by tokenizing, removing punctuation and stopwords.\"\"\"\n",
+    "    words = word_tokenize(text)\n",
+    "    stop_words = set(stopwords.words('english'))\n",
+    "    return [word.lower() for word in words if word.isalnum() and word.lower() not in stop_words]\n",
+    "\n",
+    "def calculate_mean_probability(bigram_freq: FreqDist, total_bigrams: int) -> dict:\n",
+    "    \"\"\"Calculates the mean probability (μ-value) of each bigram.\"\"\"\n",
+    "    mean_probabilities = {bigram: freq / total_bigrams for bigram, freq in bigram_freq.items()}\n",
+    "    return mean_probabilities\n",
+    "\n",
+    "def perform_statistical_tests(bigram_freq: FreqDist, word_freq: FreqDist, total_bigrams: int):\n",
+    "    \"\"\"Perform t-test and chi-square test for each bigram.\"\"\"\n",
+    "    results = []\n",
+    "\n",
+    "    for bigram, observed_freq in bigram_freq.items():\n",
+    "        word1, word2 = bigram\n",
+    "        freq_w1 = word_freq.get(word1, 0)\n",
+    "        freq_w2 = word_freq.get(word2, 0)\n",
+    "        \n",
+    "        # Expected frequency for the bigram assuming independence\n",
+    "        expected_freq = (freq_w1 * freq_w2) / total_bigrams\n",
+    "        \n",
+    "        # Chi-square test\n",
+    "        observed = np.array([\n",
+    "            [observed_freq, freq_w1 - observed_freq],\n",
+    "            [freq_w2 - observed_freq, total_bigrams - (freq_w1 + freq_w2 - observed_freq)]\n",
+    "        ])\n",
+    "        \n",
+    "        try:\n",
+    "            chi2_stat, p_value_chi2, dof, ex = chi2_contingency(observed)\n",
+    "        except ValueError:\n",
+    "            chi2_stat, p_value_chi2 = np.nan, np.nan\n",
+    "        \n",
+    "        # Perform one-sample t-test\n",
+    "        sample_mean = observed_freq\n",
+    "        sample_std = np.std([observed_freq] * 10)  # Simulating 10 observations\n",
+    "        t_stat, p_value_t = ttest_1samp([sample_mean], expected_freq)\n",
+    "        \n",
+    "        results.append((bigram, observed_freq, t_stat, p_value_t, chi2_stat, p_value_chi2))\n",
+    "    \n",
+    "    return results\n",
+    "\n",
+    "def main():\n",
+    "    # Download required NLTK data\n",
+    "    # nltk.download('punkt')\n",
+    "    # nltk.download('stopwords')\n",
+    "\n",
+    "    # Load text\n",
+    "    with open(\"text3.txt\", 'r') as file:\n",
+    "        text = file.read()\n",
+    "\n",
+    "    # Preprocess text\n",
+    "    words = preprocess_text(text)\n",
+    "\n",
+    "    # Calculate word frequency distribution\n",
+    "    word_freq = FreqDist(words)\n",
+    "\n",
+    "    # Calculate bigrams and their frequencies\n",
+    "    bigrams = list(nltk.bigrams(words))\n",
+    "    bigram_freq = FreqDist(bigrams)\n",
+    "\n",
+    "    # Calculate mean probability (μ-value) for each bigram\n",
+    "    mean_probabilities = calculate_mean_probability(bigram_freq, len(bigrams))\n",
+    "\n",
+    "    # Sort collocations by mean probability\n",
+    "    collocations = sorted(mean_probabilities.items(), key=lambda x: x[1], reverse=True)\n",
+    "\n",
+    "    # Calculate mean of bigram frequencies\n",
+    "    total_bigram_freq = sum(bigram_freq.values())\n",
+    "    mean_bigram_freq = total_bigram_freq / len(bigram_freq)\n",
+    "\n",
+    "    # Print mean bigram frequency\n",
+    "    print(f\"Mean Bigram Frequency: {mean_bigram_freq:.2f}\\n\")\n",
+    "\n",
+    "    # Perform statistical tests for each bigram\n",
+    "    results = perform_statistical_tests(bigram_freq, word_freq, len(bigrams))\n",
+    "\n",
+    "    # Print top N collocations with their frequencies and mean probabilities\n",
+    "    N = 10\n",
+    "    headers = [\"Rank\", \"Bigram\", \"Frequency\", \"Mean Probability (μ-value)\", \"t-Statistic\", \"p-Value (t-Test)\", \"Chi2 Statistic\", \"p-Value (Chi-Square)\"]\n",
+    "    table = []\n",
+    "    for i, (bigram, observed_freq, t_stat, p_value_t, chi2_stat, p_value_chi2) in enumerate(results[:N]):\n",
+    "        table.append([\n",
+    "            i+1, \n",
+    "            bigram, \n",
+    "            observed_freq, \n",
+    "            f\"{mean_probabilities.get(bigram, 0):.6f}\", \n",
+    "            f\"{t_stat:.4f}\", \n",
+    "            f\"{p_value_t:.4f}\", \n",
+    "            f\"{chi2_stat:.4f}\", \n",
+    "            f\"{p_value_chi2:.4f}\"\n",
+    "        ])\n",
+    "    \n",
+    "    print(tabulate(table, headers, tablefmt=\"orgtbl\"))\n",
+    "\n",
+    "if __name__ == \"__main__\":\n",
+    "    main()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "9b83ea08-3cf4-47db-8a13-aed1d7b21d6d",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Mean Bigram Frequency: 1.17\n",
+      "\n",
+      "+--------+--------------------------------+-------------+----------------+---------------+--------------------+------------------+------------------------+\n",
+      "|   Rank | Bigram                         |   Frequency |   Mean Prob(μ) |   t-Statistic |   p-Value (t-Test) |   Chi2 Statistic |   p-Value (Chi-Square) |\n",
+      "+========+================================+=============+================+===============+====================+==================+========================+\n",
+      "|      1 | ('impact', 'artificial')       |           2 |       0.002699 |           nan |                nan |         183.747  |                 0      |\n",
+      "+--------+--------------------------------+-------------+----------------+---------------+--------------------+------------------+------------------------+\n",
+      "|      2 | ('artificial', 'intelligence') |           3 |       0.004049 |           nan |                nan |         307.077  |                 0      |\n",
+      "+--------+--------------------------------+-------------+----------------+---------------+--------------------+------------------+------------------------+\n",
+      "|      3 | ('intelligence', 'data')       |           3 |       0.004049 |           nan |                nan |          12.4097 |                 0.0004 |\n",
+      "+--------+--------------------------------+-------------+----------------+---------------+--------------------+------------------+------------------------+\n",
+      "|      4 | ('data', 'science')            |          15 |       0.020243 |           nan |                nan |         167.483  |                 0      |\n",
+      "+--------+--------------------------------+-------------+----------------+---------------+--------------------+------------------+------------------------+\n",
+      "|      5 | ('science', 'introduction')    |           1 |       0.00135  |           nan |                nan |          11.6208 |                 0.0007 |\n",
+      "+--------+--------------------------------+-------------+----------------+---------------+--------------------+------------------+------------------------+\n",
+      "|      6 | ('introduction', 'recent')     |           1 |       0.00135  |           nan |                nan |         184.75   |                 0      |\n",
+      "+--------+--------------------------------+-------------+----------------+---------------+--------------------+------------------+------------------------+\n",
+      "|      7 | ('recent', 'years')            |           1 |       0.00135  |           nan |                nan |         184.75   |                 0      |\n",
+      "+--------+--------------------------------+-------------+----------------+---------------+--------------------+------------------+------------------------+\n",
+      "|      8 | ('years', 'convergence')       |           1 |       0.00135  |           nan |                nan |         184.75   |                 0      |\n",
+      "+--------+--------------------------------+-------------+----------------+---------------+--------------------+------------------+------------------------+\n",
+      "|      9 | ('convergence', 'artificial')  |           1 |       0.00135  |           nan |                nan |          61.0835 |                 0      |\n",
+      "+--------+--------------------------------+-------------+----------------+---------------+--------------------+------------------+------------------------+\n",
+      "|     10 | ('intelligence', 'ai')         |           1 |       0.00135  |           nan |                nan |           0.4959 |                 0.4813 |\n",
+      "+--------+--------------------------------+-------------+----------------+---------------+--------------------+------------------+------------------------+\n"
+     ]
+    }
+   ],
+   "source": [
+    "import nltk\n",
+    "from nltk.tokenize import word_tokenize\n",
+    "from nltk.corpus import stopwords\n",
+    "from nltk.probability import FreqDist\n",
+    "from tabulate import tabulate\n",
+    "from scipy.stats import ttest_1samp, chi2_contingency\n",
+    "import numpy as np\n",
+    "\n",
+    "def preprocess_text(text: str) -> list[str]:\n",
+    "    \"\"\"Preprocesses the text by tokenizing, removing punctuation and stopwords.\"\"\"\n",
+    "    words = word_tokenize(text)\n",
+    "    stop_words = set(stopwords.words('english'))\n",
+    "    return [word.lower() for word in words if word.isalnum() and word.lower() not in stop_words]\n",
+    "\n",
+    "def calculate_mean_probability(bigram_freq: FreqDist, total_bigrams: int) -> dict:\n",
+    "    \"\"\"Calculates the mean probability (μ-value) of each bigram.\"\"\"\n",
+    "    mean_probabilities = {bigram: freq / total_bigrams for bigram, freq in bigram_freq.items()}\n",
+    "    return mean_probabilities\n",
+    "\n",
+    "def perform_statistical_tests(bigram_freq: FreqDist, word_freq: FreqDist, total_bigrams: int):\n",
+    "    \"\"\"Perform t-test and chi-square test for each bigram.\"\"\"\n",
+    "    results = []\n",
+    "\n",
+    "    for bigram, observed_freq in bigram_freq.items():\n",
+    "        word1, word2 = bigram\n",
+    "        freq_w1 = word_freq.get(word1, 0)\n",
+    "        freq_w2 = word_freq.get(word2, 0)\n",
+    "        \n",
+    "        # Expected frequency for the bigram assuming independence\n",
+    "        expected_freq = (freq_w1 * freq_w2) / total_bigrams\n",
+    "        \n",
+    "        # Chi-square test\n",
+    "        observed = np.array([\n",
+    "            [observed_freq, freq_w1 - observed_freq],\n",
+    "            [freq_w2 - observed_freq, total_bigrams - (freq_w1 + freq_w2 - observed_freq)]\n",
+    "        ])\n",
+    "        \n",
+    "        try:\n",
+    "            chi2_stat, p_value_chi2, dof, ex = chi2_contingency(observed)\n",
+    "        except ValueError:\n",
+    "            chi2_stat, p_value_chi2 = np.nan, np.nan\n",
+    "        \n",
+    "        # Perform one-sample t-test\n",
+    "        sample_mean = observed_freq\n",
+    "        sample_std = np.std([observed_freq] * 10)  # Simulating 10 observations\n",
+    "        \n",
+    "        if sample_std == 0:\n",
+    "            t_stat, p_value_t = np.nan, np.nan\n",
+    "        else:\n",
+    "            t_stat, p_value_t = ttest_1samp([sample_mean], expected_freq)\n",
+    "        \n",
+    "        results.append((bigram, observed_freq, t_stat, p_value_t, chi2_stat, p_value_chi2))\n",
+    "    \n",
+    "    return results\n",
+    "\n",
+    "def main():\n",
+    "    # Download required NLTK data\n",
+    "    #nltk.download('punkt')\n",
+    "    #nltk.download('stopwords')\n",
+    "\n",
+    "    # Load text\n",
+    "    with open(\"text3.txt\", 'r') as file:\n",
+    "        text = file.read()\n",
+    "\n",
+    "    # Preprocess text\n",
+    "    words = preprocess_text(text)\n",
+    "\n",
+    "    # Calculate word frequency distribution\n",
+    "    word_freq = FreqDist(words)\n",
+    "\n",
+    "    # Calculate bigrams and their frequencies\n",
+    "    bigrams = list(nltk.bigrams(words))\n",
+    "    bigram_freq = FreqDist(bigrams)\n",
+    "\n",
+    "    # Calculate mean probability (μ-value) for each bigram\n",
+    "    mean_probabilities = calculate_mean_probability(bigram_freq, len(bigrams))\n",
+    "\n",
+    "    # Sort collocations by mean probability\n",
+    "    collocations = sorted(mean_probabilities.items(), key=lambda x: x[1], reverse=True)\n",
+    "\n",
+    "    # Calculate mean of bigram frequencies\n",
+    "    total_bigram_freq = sum(bigram_freq.values())\n",
+    "    mean_bigram_freq = total_bigram_freq / len(bigram_freq)\n",
+    "\n",
+    "    # Print mean bigram frequency\n",
+    "    print(f\"Mean Bigram Frequency: {mean_bigram_freq:.2f}\\n\")\n",
+    "\n",
+    "    # Perform statistical tests for each bigram\n",
+    "    results = perform_statistical_tests(bigram_freq, word_freq, len(bigrams))\n",
+    "\n",
+    "    # Print top N collocations with their frequencies and mean probabilities\n",
+    "    N = 10\n",
+    "    headers = [\"Rank\", \"Bigram\", \"Frequency\", \"Mean Prob(μ)\", \"t-Statistic\", \"p-Value (t-Test)\", \"Chi2 Statistic\", \"p-Value (Chi-Square)\"]\n",
+    "    table = []\n",
+    "    for i, (bigram, observed_freq, t_stat, p_value_t, chi2_stat, p_value_chi2) in enumerate(results[:N]):\n",
+    "        table.append([\n",
+    "            i+1, \n",
+    "            bigram, \n",
+    "            observed_freq, \n",
+    "            f\"{mean_probabilities.get(bigram, 0):.6f}\", \n",
+    "            f\"{t_stat:.4f}\" if not np.isnan(t_stat) else \"NaN\", \n",
+    "            f\"{p_value_t:.4f}\" if not np.isnan(p_value_t) else \"NaN\", \n",
+    "            f\"{chi2_stat:.4f}\" if not np.isnan(chi2_stat) else \"NaN\", \n",
+    "            f\"{p_value_chi2:.4f}\" if not np.isnan(p_value_chi2) else \"NaN\"\n",
+    "        ])\n",
+    "    print(tabulate(table, headers, tablefmt=\"grid\"))\n",
+    "\n",
+    "if __name__ == \"__main__\":\n",
+    "    main()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "841d503b-6e3c-4c37-b11f-5482cc462cad",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Mean Bigram Frequency: 1.17\n",
+      "\n",
+      "+--------+--------------------------------+-------------+----------------+---------------+-------------------+--------------+-----------------------+\n",
+      "|   Rank | Bigram                         |   Frequency |   Mean Prob(μ) |   t-Statistic |   p-Value(t-Test) |   Chi Square |   p-Value(Chi-Square) |\n",
+      "+========+================================+=============+================+===============+===================+==============+=======================+\n",
+      "|      1 | ('impact', 'artificial')       |           2 |       0.002699 |        1.4152 |            0.1574 |     183.747  |                0      |\n",
+      "+--------+--------------------------------+-------------+----------------+---------------+-------------------+--------------+-----------------------+\n",
+      "|      2 | ('artificial', 'intelligence') |           3 |       0.004049 |        1.7344 |            0.0833 |     307.077  |                0      |\n",
+      "+--------+--------------------------------+-------------+----------------+---------------+-------------------+--------------+-----------------------+\n",
+      "|      3 | ('intelligence', 'data')       |           3 |       0.004049 |        1.7344 |            0.0833 |      12.4097 |                0.0004 |\n",
+      "+--------+--------------------------------+-------------+----------------+---------------+-------------------+--------------+-----------------------+\n",
+      "|      4 | ('data', 'science')            |          15 |       0.020243 |        3.9101 |            0.0001 |     167.483  |                0      |\n",
+      "+--------+--------------------------------+-------------+----------------+---------------+-------------------+--------------+-----------------------+\n",
+      "|      5 | ('science', 'introduction')    |           1 |       0.00135  |        1      |            0.3176 |      11.6208 |                0.0007 |\n",
+      "+--------+--------------------------------+-------------+----------------+---------------+-------------------+--------------+-----------------------+\n",
+      "|      6 | ('introduction', 'recent')     |           1 |       0.00135  |        1      |            0.3176 |     184.75   |                0      |\n",
+      "+--------+--------------------------------+-------------+----------------+---------------+-------------------+--------------+-----------------------+\n",
+      "|      7 | ('recent', 'years')            |           1 |       0.00135  |        1      |            0.3176 |     184.75   |                0      |\n",
+      "+--------+--------------------------------+-------------+----------------+---------------+-------------------+--------------+-----------------------+\n",
+      "|      8 | ('years', 'convergence')       |           1 |       0.00135  |        1      |            0.3176 |     184.75   |                0      |\n",
+      "+--------+--------------------------------+-------------+----------------+---------------+-------------------+--------------+-----------------------+\n",
+      "|      9 | ('convergence', 'artificial')  |           1 |       0.00135  |        1      |            0.3176 |      61.0835 |                0      |\n",
+      "+--------+--------------------------------+-------------+----------------+---------------+-------------------+--------------+-----------------------+\n",
+      "|     10 | ('intelligence', 'ai')         |           1 |       0.00135  |        1      |            0.3176 |       0.4959 |                0.4813 |\n",
+      "+--------+--------------------------------+-------------+----------------+---------------+-------------------+--------------+-----------------------+\n"
+     ]
+    }
+   ],
+   "source": [
+    "import nltk\n",
+    "from nltk.tokenize import word_tokenize\n",
+    "from nltk.corpus import stopwords\n",
+    "from nltk.probability import FreqDist\n",
+    "from tabulate import tabulate\n",
+    "from scipy.stats import chi2_contingency, ttest_1samp\n",
+    "import numpy as np\n",
+    "\n",
+    "def preprocess_text(text: str) -> list[str]:\n",
+    "    \"\"\"Preprocesses the text by tokenizing, removing punctuation and stopwords.\"\"\"\n",
+    "    words = word_tokenize(text)\n",
+    "    stop_words = set(stopwords.words('english'))\n",
+    "    return [word.lower() for word in words if word.isalnum() and word.lower() not in stop_words]\n",
+    "\n",
+    "def calculate_mean_probability(bigram_freq: FreqDist, total_bigrams: int) -> dict:\n",
+    "    \"\"\"Calculates the mean probability (μ-value) of each bigram.\"\"\"\n",
+    "    mean_probabilities = {bigram: freq / total_bigrams for bigram, freq in bigram_freq.items()}\n",
+    "    return mean_probabilities\n",
+    "\n",
+    "def perform_statistical_tests(bigram_freq: FreqDist, word_freq: FreqDist, total_bigrams: int):\n",
+    "    \"\"\"Perform t-test and chi-square test for each bigram.\"\"\"\n",
+    "    results = []\n",
+    "\n",
+    "    for bigram, observed_freq in bigram_freq.items():\n",
+    "        word1, word2 = bigram\n",
+    "        freq_w1 = word_freq.get(word1, 0)\n",
+    "        freq_w2 = word_freq.get(word2, 0)\n",
+    "        \n",
+    "        # Expected frequency for the bigram assuming independence\n",
+    "        expected_freq = (freq_w1 * freq_w2) / total_bigrams\n",
+    "        \n",
+    "        # Chi-square test\n",
+    "        observed = np.array([\n",
+    "            [observed_freq, freq_w1 - observed_freq],\n",
+    "            [freq_w2 - observed_freq, total_bigrams - (freq_w1 + freq_w2 - observed_freq)]\n",
+    "        ])\n",
+    "        \n",
+    "        try:\n",
+    "            chi2_stat, p_value_chi2, dof, ex = chi2_contingency(observed)\n",
+    "        except ValueError:\n",
+    "            chi2_stat, p_value_chi2 = np.nan, np.nan\n",
+    "        \n",
+    "        # Generate sample data to perform t-test\n",
+    "        sample_data = [observed_freq] * observed_freq + [expected_freq] * (total_bigrams - observed_freq)\n",
+    "        \n",
+    "        # Perform one-sample t-test\n",
+    "        t_stat, p_value_t = ttest_1samp(sample_data, expected_freq)\n",
+    "        \n",
+    "        results.append((bigram, observed_freq, t_stat, p_value_t, chi2_stat, p_value_chi2))\n",
+    "    \n",
+    "    return results\n",
+    "\n",
+    "def main():\n",
+    "    # Download required NLTK data\n",
+    "    # nltk.download('punkt')\n",
+    "    # nltk.download('stopwords')\n",
+    "\n",
+    "    # Load text\n",
+    "    with open(\"text3.txt\", 'r') as file:\n",
+    "        text = file.read()\n",
+    "\n",
+    "    # Preprocess text\n",
+    "    words = preprocess_text(text)\n",
+    "\n",
+    "    # Calculate word frequency distribution\n",
+    "    word_freq = FreqDist(words)\n",
+    "\n",
+    "    # Calculate bigrams and their frequencies\n",
+    "    bigrams = list(nltk.bigrams(words))\n",
+    "    bigram_freq = FreqDist(bigrams)\n",
+    "\n",
+    "    # Calculate mean probability (μ-value) for each bigram\n",
+    "    mean_probabilities = calculate_mean_probability(bigram_freq, len(bigrams))\n",
+    "\n",
+    "    # Sort collocations by mean probability\n",
+    "    collocations = sorted(mean_probabilities.items(), key=lambda x: x[1], reverse=True)\n",
+    "\n",
+    "    # Calculate mean of bigram frequencies\n",
+    "    total_bigram_freq = sum(bigram_freq.values())\n",
+    "    mean_bigram_freq = total_bigram_freq / len(bigram_freq)\n",
+    "\n",
+    "    # Print mean bigram frequency\n",
+    "    print(f\"Mean Bigram Frequency: {mean_bigram_freq:.2f}\\n\")\n",
+    "\n",
+    "    # Perform statistical tests for each bigram\n",
+    "    results = perform_statistical_tests(bigram_freq, word_freq, len(bigrams))\n",
+    "\n",
+    "    # Print top N collocations with their frequencies and mean probabilities\n",
+    "    N = 10\n",
+    "    headers = [\"Rank\", \"Bigram\", \"Frequency\", \"Mean Prob(μ)\", \"t-Statistic\", \"p-Value(t-Test)\", \"Chi Square\", \"p-Value(Chi-Square)\"]\n",
+    "    table = []\n",
+    "    for i, (bigram, observed_freq, t_stat, p_value_t, chi2_stat, p_value_chi2) in enumerate(results[:N]):\n",
+    "        table.append([\n",
+    "            i + 1,\n",
+    "            bigram,\n",
+    "            observed_freq,\n",
+    "            f\"{mean_probabilities.get(bigram, 0):.6f}\",\n",
+    "            f\"{t_stat:.4f}\" if not np.isnan(t_stat) else \"NaN\",\n",
+    "            f\"{p_value_t:.4f}\" if not np.isnan(p_value_t) else \"NaN\",\n",
+    "            f\"{chi2_stat:.4f}\" if not np.isnan(chi2_stat) else \"NaN\",\n",
+    "            f\"{p_value_chi2:.4f}\" if not np.isnan(p_value_chi2) else \"NaN\"\n",
+    "        ])\n",
+    "    print(tabulate(table, headers, tablefmt=\"grid\"))\n",
+    "\n",
+    "if __name__ == \"__main__\":\n",
+    "    main()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fcb18fd4-7444-4f9e-881e-279099049c9f",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}