PgsFile 0.4.2__py3-none-any.whl → 0.4.4__py3-none-any.whl

PgsFile/PgsFile.py

@@ -3953,6 +3953,43 @@ def extract_keywords_en(target_text, top_n=10):
     # Return top N keywords
     return keyword_scores[:top_n]
 
+def extract_keywords_en_be21(target_text, top_n=10):
+    """Extract keywords from target text using log-likelihood with absolute reference frequencies."""
+    # Example usage
+    my_dic_path = get_library_location("PgsFile")+"/PgsFile/models/dics/BE21.json" # BE21_wordlist
+    reference_freq = get_data_json(my_dic_path)
+    # Tokenize target text and preserve original case
+    original_words = word_tokenize2(target_text)
+    lower_words = [w.lower() for w in original_words if w.lower() not in BigPunctuation and w.lower() not in get_stopwords()]
+    total_target = len(lower_words)
+
+    # Calculate target word frequencies
+    target_word_freq = defaultdict(int)
+    word_case_mapping = {}
+    for orig_word, lower_word in zip(original_words, [w.lower() for w in original_words]):
+        if lower_word in lower_words:
+            target_word_freq[lower_word] += 1
+            if lower_word not in word_case_mapping:
+                word_case_mapping[lower_word] = orig_word
+
+    # Calculate total reference frequency
+    total_reference = sum(reference_freq.values())
+
+    # Calculate log-likelihood for each word
+    keyword_scores = []
+    for word, target_count in target_word_freq.items():
+        reference_count = reference_freq.get(word, 0)
+        ll = calculate_log_likelihood(target_count, reference_count, total_target, total_reference)
+        relative_freq = target_count / total_target
+        original_word = word_case_mapping.get(word, word)
+        keyword_scores.append((original_word, target_count, relative_freq, ll))
+
+    # Sort keywords by log-likelihood score
+    keyword_scores.sort(key=lambda x: x[3], reverse=True)
+
+    # Return top N keywords
+    return keyword_scores[:top_n]
+
 def resize_image(input_image_path, output_image_path, max_size_kb):
     '''
     # Example 1: Resizing a JPG image

PgsFile/__init__.py

@@ -49,7 +49,7 @@ from .PgsFile import ngrams, bigrams, trigrams, everygrams, compute_similarity,
 from .PgsFile import word_list, batch_word_list
 from .PgsFile import cs, cs1, sent_tokenize, word_tokenize, word_tokenize2
 from .PgsFile import word_lemmatize, word_POS, word_NER
-from .PgsFile import extract_noun_phrases, get_LLMs_prompt, extract_keywords_en
+from .PgsFile import extract_noun_phrases, get_LLMs_prompt, extract_keywords_en, extract_keywords_en_be21
 from .PgsFile import extract_dependency_relations, extract_dependency_relations_full
 from .PgsFile import predict_category