PgsFile 0.4.6__py3-none-any.whl → 0.4.7__py3-none-any.whl

PgsFile/PgsFile.py

@@ -4090,4 +4090,55 @@ def markdown_to_python_object(data):
         return ast.literal_eval(code_str)
     except Exception:
         return code_str.strip()
-    
+
+
+import math
+from collections import defaultdict, Counter
+
+def tfidf_keyword_extraction(documents, top_percent=(0.0, 0.10)):
+    """
+    Extract keywords from a small set of tokenized documents using TF-IDF.
+
+    Parameters
+    ----------
+    documents : list of list of str
+        Corpus represented as tokenized documents.
+    top_percent : tuple of float
+        Range of percentage (low, high) to select top keyword candidates.
+
+    Returns
+    -------
+    full_list : list of tuple
+        All (term, tf-idf_score) sorted by score in descending order.
+    candidates : list of tuple
+        Keyword candidates from top_10% range.
+    """
+    log = math.log  # local reference for speed
+
+    # Step 1: Compute IDF
+    total_docs = len(documents)
+    doc_freq = defaultdict(int)
+    for doc in documents:
+        for term in set(doc):
+            doc_freq[term] += 1
+    idf = {term: log((total_docs + 1) / (df + 1)) + 1 for term, df in doc_freq.items()}
+
+    # Step 2: Compute TF-IDF
+    tfidf_scores = {}
+    for doc in documents:
+        total_terms = len(doc)
+        term_counts = Counter(doc)
+        for term, count in term_counts.items():
+            tfidf_scores[term] = (count / total_terms) * idf[term]  # overwrite as before
+
+    # Step 3: Sort full list
+    full_list = sorted(tfidf_scores.items(), key=lambda x: x[1], reverse=True)
+
+    # Step 4: Extract candidates based on percentage range
+    n_terms = len(full_list)
+    low_cut = int(n_terms * top_percent[0])
+    high_cut = int(n_terms * top_percent[1])
+    candidates = full_list[low_cut:high_cut]  # slice range
+
+    return full_list, candidates
+

PgsFile/__init__.py

@@ -53,6 +53,7 @@ from .PgsFile import word_lemmatize, word_POS, word_NER
 from .PgsFile import extract_noun_phrases, get_LLMs_prompt, extract_keywords_en, extract_keywords_en_be21
 from .PgsFile import extract_dependency_relations, extract_dependency_relations_full
 from .PgsFile import predict_category
+from .PgsFile import tfidf_keyword_extraction
 
 # 8. Maths
 from .PgsFile import len_rows, check_empty_cells

{PgsFile-0.4.6.dist-info → PgsFile-0.4.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PgsFile
-Version: 0.4.6
+Version: 0.4.7
 Summary: This module simplifies Python package management, script execution, file handling, web scraping, and multimedia downloads. The module supports (LLM-based) NLP tasks such as OCR, tokenization, lemmatization, POS tagging, NER, ATE, dependency parsing, MDD, WSD, LIWC, and MIP analysis. It also generates word lists, and plots data, aiding literary students. Ideal for scraping data, cleaning text, and analyzing language, it offers user-friendly tools to streamline workflows.
 Home-page: https://mp.weixin.qq.com/s/lWMkYDWQMjBJNKY2vMYTpw
 Author: Pan Guisheng

{PgsFile-0.4.6.dist-info → PgsFile-0.4.7.dist-info}/RECORD

@@ -1,5 +1,5 @@
-PgsFile/PgsFile.py,sha256=lsccmRjB-vHPTeAsGG97CYPjTiTrVnbiiSHCqaGc6sk,168231
-PgsFile/__init__.py,sha256=0yJbrdpue45cageZqejZAsjdWXcEg-Cs2NT7elOF1rQ,3627
+PgsFile/PgsFile.py,sha256=3iyfFE5THgwwz0_MtylUlMn-72gsRaCbUxdm9LcI8nQ,169903
+PgsFile/__init__.py,sha256=mWZ8dfTlzeCfTHFlWyHY3vCwqyM4_YQBGPd6vBoNGso,3674
 PgsFile/Corpora/Idioms/English_Idioms_8774.txt,sha256=qlsP0yI_XGECBRiPZuLkGZpdasc77sWSKexANu7v8_M,175905
 PgsFile/Corpora/Monolingual/Chinese/People's Daily 20130605/Raw/00000000.txt,sha256=SLGGSMSb7Ff1RoBstsTW3yX2wNZpqEUchFNpcI-mrR4,1513
 PgsFile/Corpora/Monolingual/Chinese/People's Daily 20130605/Raw/00000001.txt,sha256=imOa6UoCOIZoPXT4_HNHgCUJtd4FTIdk2FZNHNBgJyg,3372
@@ -2591,8 +2591,8 @@ PgsFile/models/prompts/3. ICTCLAS Prompt.txt,sha256=VFn6N_JViAbyy9NazA8gjX6SGo5m
 PgsFile/models/prompts/4. OCR prompt.txt,sha256=YxUQ2IlE52k0fcBnGsuOHqWAmfiEmIu6iRz5zecQ8dk,260
 PgsFile/models/prompts/5. ATE prompt.txt,sha256=5wu0gGlsV7DI0LruYM3-uAC6brppyYD0IoiFVjMqm5Y,1553
 PgsFile/models/prompts/6. ATE3 prompt.txt,sha256=VnaXpPa6BgZHUcm8PxmP_qgU-8xEoTB3XcBqjwCUy_g,1254
-PgsFile-0.4.6.dist-info/LICENSE,sha256=cE5c-QToSkG1KTUsU8drQXz1vG0EbJWuU4ybHTRb5SE,1138
-PgsFile-0.4.6.dist-info/METADATA,sha256=eraBURV9-2LJqn0sTWNW35L286E1r8DXVX-ZVAHurY4,2994
-PgsFile-0.4.6.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-PgsFile-0.4.6.dist-info/top_level.txt,sha256=028hCfwhF3UpfD6X0rwtWpXI1RKSTeZ1ALwagWaSmX8,8
-PgsFile-0.4.6.dist-info/RECORD,,
+PgsFile-0.4.7.dist-info/LICENSE,sha256=cE5c-QToSkG1KTUsU8drQXz1vG0EbJWuU4ybHTRb5SE,1138
+PgsFile-0.4.7.dist-info/METADATA,sha256=AUDP0QL-_-2YKDZyH6i5dz7DrFmJO50TuGr5QqnkQhw,2994
+PgsFile-0.4.7.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+PgsFile-0.4.7.dist-info/top_level.txt,sha256=028hCfwhF3UpfD6X0rwtWpXI1RKSTeZ1ALwagWaSmX8,8
+PgsFile-0.4.7.dist-info/RECORD,,