PgsFile 0.4.5__py3-none-any.whl → 0.4.7__py3-none-any.whl

PgsFile/PgsFile.py

@@ -4090,4 +4090,55 @@ def markdown_to_python_object(data):
         return ast.literal_eval(code_str)
     except Exception:
         return code_str.strip()
-    
+
+
+import math
+from collections import defaultdict, Counter
+
+def tfidf_keyword_extraction(documents, top_percent=(0.0, 0.10)):
+    """
+    Extract keywords from a small set of tokenized documents using TF-IDF.
+
+    Parameters
+    ----------
+    documents : list of list of str
+        Corpus represented as tokenized documents.
+    top_percent : tuple of float
+        Range of percentage (low, high) to select top keyword candidates.
+
+    Returns
+    -------
+    full_list : list of tuple
+        All (term, tf-idf_score) sorted by score in descending order.
+    candidates : list of tuple
+        Keyword candidates from top_10% range.
+    """
+    log = math.log  # local reference for speed
+
+    # Step 1: Compute IDF
+    total_docs = len(documents)
+    doc_freq = defaultdict(int)
+    for doc in documents:
+        for term in set(doc):
+            doc_freq[term] += 1
+    idf = {term: log((total_docs + 1) / (df + 1)) + 1 for term, df in doc_freq.items()}
+
+    # Step 2: Compute TF-IDF
+    tfidf_scores = {}
+    for doc in documents:
+        total_terms = len(doc)
+        term_counts = Counter(doc)
+        for term, count in term_counts.items():
+            tfidf_scores[term] = (count / total_terms) * idf[term]  # overwrite as before
+
+    # Step 3: Sort full list
+    full_list = sorted(tfidf_scores.items(), key=lambda x: x[1], reverse=True)
+
+    # Step 4: Extract candidates based on percentage range
+    n_terms = len(full_list)
+    low_cut = int(n_terms * top_percent[0])
+    high_cut = int(n_terms * top_percent[1])
+    candidates = full_list[low_cut:high_cut]  # slice range
+
+    return full_list, candidates
+

PgsFile/__init__.py

@@ -53,6 +53,7 @@ from .PgsFile import word_lemmatize, word_POS, word_NER
 from .PgsFile import extract_noun_phrases, get_LLMs_prompt, extract_keywords_en, extract_keywords_en_be21
 from .PgsFile import extract_dependency_relations, extract_dependency_relations_full
 from .PgsFile import predict_category
+from .PgsFile import tfidf_keyword_extraction
 
 # 8. Maths
 from .PgsFile import len_rows, check_empty_cells

PgsFile/models/prompts/5. ATE prompt.txt

@@ -3,6 +3,7 @@ You are an excellent automatic term extraction (ATE) system. Your task is to ide
 What are the terms in the following text? Terms should not include named entities.
 Output Format: ["list of terms present"]
 If no terms are presented, keep it an empty list: []
+Do not include any explanations or additional text in the output.
 
 EXAMPLES:
 

PgsFile/models/prompts/6. ATE3 prompt.txt

@@ -4,8 +4,9 @@ Extract terms from the input text and classify each term into one of the followi
 •	OOD_Term: Lexicon-specific but not domain-specific (known mainly by experts, but not relevant to the domain)
 
 Terms should not include named entities.
-Output Format (a list of python-tuple): [("2-vessel cad", "Specific_Term"), ("aortic valve", "Common_Term"), ("p-value", "OOD_Term")]
+Output Format (a list of python-list): [["2-vessel cad", "Specific_Term"], ["aortic valve", "Common_Term"], ["p-value", "OOD_Term"]]
 If no terms are presented, keep it an empty list: []
+Do not include any explanations or additional text in the output.
 
 Examples (in the domain of heart failure):
 •	"ejection fraction" → "Specific_Term": laypersons generally do not know what it means, and it is strongly related to the domain.
@@ -15,4 +16,4 @@ Examples (in the domain of heart failure):
 Now, please extract the terms from the following sentence:
 
 Sentence: "{sentence}"
-Output: ["list of terms present"]
+Output: [[]]

{PgsFile-0.4.5.dist-info → PgsFile-0.4.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PgsFile
-Version: 0.4.5
+Version: 0.4.7
 Summary: This module simplifies Python package management, script execution, file handling, web scraping, and multimedia downloads. The module supports (LLM-based) NLP tasks such as OCR, tokenization, lemmatization, POS tagging, NER, ATE, dependency parsing, MDD, WSD, LIWC, and MIP analysis. It also generates word lists, and plots data, aiding literary students. Ideal for scraping data, cleaning text, and analyzing language, it offers user-friendly tools to streamline workflows.
 Home-page: https://mp.weixin.qq.com/s/lWMkYDWQMjBJNKY2vMYTpw
 Author: Pan Guisheng

{PgsFile-0.4.5.dist-info → PgsFile-0.4.7.dist-info}/RECORD

@@ -1,5 +1,5 @@
-PgsFile/PgsFile.py,sha256=lsccmRjB-vHPTeAsGG97CYPjTiTrVnbiiSHCqaGc6sk,168231
-PgsFile/__init__.py,sha256=0yJbrdpue45cageZqejZAsjdWXcEg-Cs2NT7elOF1rQ,3627
+PgsFile/PgsFile.py,sha256=3iyfFE5THgwwz0_MtylUlMn-72gsRaCbUxdm9LcI8nQ,169903
+PgsFile/__init__.py,sha256=mWZ8dfTlzeCfTHFlWyHY3vCwqyM4_YQBGPd6vBoNGso,3674
 PgsFile/Corpora/Idioms/English_Idioms_8774.txt,sha256=qlsP0yI_XGECBRiPZuLkGZpdasc77sWSKexANu7v8_M,175905
 PgsFile/Corpora/Monolingual/Chinese/People's Daily 20130605/Raw/00000000.txt,sha256=SLGGSMSb7Ff1RoBstsTW3yX2wNZpqEUchFNpcI-mrR4,1513
 PgsFile/Corpora/Monolingual/Chinese/People's Daily 20130605/Raw/00000001.txt,sha256=imOa6UoCOIZoPXT4_HNHgCUJtd4FTIdk2FZNHNBgJyg,3372
@@ -2589,10 +2589,10 @@ PgsFile/models/prompts/1. MIP prompt.txt,sha256=4lHlHmleayRytqr1n9jtt6vn1rQvyf4B
 PgsFile/models/prompts/2. WSD prompt.txt,sha256=o-ZFtCRUCDrXgm040WTQch9v2Y_r2SIlrZaquilJjgQ,2348
 PgsFile/models/prompts/3. ICTCLAS Prompt.txt,sha256=VFn6N_JViAbyy9NazA8gjX6SGo5mgBcZOf95aC9JB84,592
 PgsFile/models/prompts/4. OCR prompt.txt,sha256=YxUQ2IlE52k0fcBnGsuOHqWAmfiEmIu6iRz5zecQ8dk,260
-PgsFile/models/prompts/5. ATE prompt.txt,sha256=ZJo9BhbbUf7CVXi2Gb5DAsV_2PGgzly2I7ze0grCo2k,1486
-PgsFile/models/prompts/6. ATE3 prompt.txt,sha256=BcefVd-RM1_IjSH1t9JfRUz6qxIoYrDxORLlysT3H70,1209
-PgsFile-0.4.5.dist-info/LICENSE,sha256=cE5c-QToSkG1KTUsU8drQXz1vG0EbJWuU4ybHTRb5SE,1138
-PgsFile-0.4.5.dist-info/METADATA,sha256=trHMmN-jQGE_NxXW35ZdVyOf7XC_Q9xT5v4vMkk48rw,2994
-PgsFile-0.4.5.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-PgsFile-0.4.5.dist-info/top_level.txt,sha256=028hCfwhF3UpfD6X0rwtWpXI1RKSTeZ1ALwagWaSmX8,8
-PgsFile-0.4.5.dist-info/RECORD,,
+PgsFile/models/prompts/5. ATE prompt.txt,sha256=5wu0gGlsV7DI0LruYM3-uAC6brppyYD0IoiFVjMqm5Y,1553
+PgsFile/models/prompts/6. ATE3 prompt.txt,sha256=VnaXpPa6BgZHUcm8PxmP_qgU-8xEoTB3XcBqjwCUy_g,1254
+PgsFile-0.4.7.dist-info/LICENSE,sha256=cE5c-QToSkG1KTUsU8drQXz1vG0EbJWuU4ybHTRb5SE,1138
+PgsFile-0.4.7.dist-info/METADATA,sha256=AUDP0QL-_-2YKDZyH6i5dz7DrFmJO50TuGr5QqnkQhw,2994
+PgsFile-0.4.7.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+PgsFile-0.4.7.dist-info/top_level.txt,sha256=028hCfwhF3UpfD6X0rwtWpXI1RKSTeZ1ALwagWaSmX8,8
+PgsFile-0.4.7.dist-info/RECORD,,