PgsFile 0.4.6__py3-none-any.whl → 0.4.8__py3-none-any.whl

PgsFile/PgsFile.py

@@ -4090,4 +4090,55 @@ def markdown_to_python_object(data):
         return ast.literal_eval(code_str)
     except Exception:
         return code_str.strip()
-    
+
+
+import math
+from collections import defaultdict, Counter
+
+def tfidf_keyword_extraction(documents, top_percent=(0.0, 0.10)):
+    """
+    Extract keywords from a small set of tokenized documents using TF-IDF.
+
+    Parameters
+    ----------
+    documents : list of list of str
+        Corpus represented as tokenized documents.
+    top_percent : tuple of float
+        Range of percentage (low, high) to select top keyword candidates.
+
+    Returns
+    -------
+    full_list : list of tuple
+        All (term, tf-idf_score) sorted by score in descending order.
+    candidates : list of tuple
+        Keyword candidates from top_10% range.
+    """
+    log = math.log  # local reference for speed
+
+    # Step 1: Compute IDF
+    total_docs = len(documents)
+    doc_freq = defaultdict(int)
+    for doc in documents:
+        for term in set(doc):
+            doc_freq[term] += 1
+    idf = {term: log((total_docs + 1) / (df + 1)) + 1 for term, df in doc_freq.items()}
+
+    # Step 2: Compute TF-IDF
+    tfidf_scores = {}
+    for doc in documents:
+        total_terms = len(doc)
+        term_counts = Counter(doc)
+        for term, count in term_counts.items():
+            tfidf_scores[term] = (count / total_terms) * idf[term]  # overwrite as before
+
+    # Step 3: Sort full list
+    full_list = sorted(tfidf_scores.items(), key=lambda x: x[1], reverse=True)
+
+    # Step 4: Extract candidates based on percentage range
+    n_terms = len(full_list)
+    low_cut = int(n_terms * top_percent[0])
+    high_cut = int(n_terms * top_percent[1])
+    candidates = full_list[low_cut:high_cut]  # slice range
+
+    return full_list, candidates
+

PgsFile/__init__.py

@@ -53,6 +53,7 @@ from .PgsFile import word_lemmatize, word_POS, word_NER
 from .PgsFile import extract_noun_phrases, get_LLMs_prompt, extract_keywords_en, extract_keywords_en_be21
 from .PgsFile import extract_dependency_relations, extract_dependency_relations_full
 from .PgsFile import predict_category
+from .PgsFile import tfidf_keyword_extraction
 
 # 8. Maths
 from .PgsFile import len_rows, check_empty_cells

PgsFile/models/prompts/7. SentAlign prompt.txt

@@ -0,0 +1,35 @@
+You are an expert in aligning Chinese and English sentences.
+Align Chinese sentences with their corresponding English translations from the input text.
+
+Priority:
+•	First attempt 1–1 alignment (one Chinese sentence ↔ one English sentence) whenever possible.
+•	If no exact 1–1 match is possible, use 1–many (one Chinese sentence ↔ multiple English sentences) or many–1 (multiple Chinese sentences ↔ one English sentence).
+•	Ensure every Chinese sentence is included in the alignment—none should be skipped.
+
+Input:
+•	src: Chinese passage
+•	tgt: English passage
+
+Output format (Python-list):
+A python List of aligned pairs without extra explanation:
+[aligned_src_sent(s)1, aligned_tgt_sent(s)1]
+
+Examples:
+Example 1 (1–1):
+src: "两年以后，大兴安岭。"
+tgt: "Two years later, the Greater Khingan Mountains."
+output: ["两年以后，大兴安岭。", "Two years later, the Greater Khingan Mountains."]
+Example 2 (1–many):
+src: "他沉默了一会儿，然后笑了。"
+tgt: "He was silent for a moment. Then he smiled."
+output: [["他沉默了一会儿，然后笑了。"], ["He was silent for a moment.", "Then he smiled."]]
+Example 3 (many–1):
+src: "风起了。天色渐暗。"
+tgt: "The wind picked up as the sky darkened."
+output: [["风起了。", "天色渐暗。"], ["The wind picked up as the sky darkened."]]
+
+Task:
+Using the priority rules above, align the following src and tgt sentences accurately:
+Source Text: {src}
+Target Text: {tgt}
+Output: ["",""]

{PgsFile-0.4.6.dist-info → PgsFile-0.4.8.dist-info}/METADATA

@@ -1,7 +1,7 @@
 Metadata-Version: 2.1
 Name: PgsFile
-Version: 0.4.6
-Summary: This module simplifies Python package management, script execution, file handling, web scraping, and multimedia downloads. The module supports (LLM-based) NLP tasks such as OCR, tokenization, lemmatization, POS tagging, NER, ATE, dependency parsing, MDD, WSD, LIWC, and MIP analysis. It also generates word lists, and plots data, aiding literary students. Ideal for scraping data, cleaning text, and analyzing language, it offers user-friendly tools to streamline workflows.
+Version: 0.4.8
+Summary: This module simplifies Python package management, script execution, file handling, web scraping, and multimedia downloads. The module supports (LLM-based) NLP tasks such as OCR, tokenization, lemmatization, POS tagging, NER, ATE, dependency parsing, MDD, WSD, LIWC, MIP analysis and Chinese-English sentence alignment. It also generates word lists, and plots data, aiding literary students. Ideal for scraping data, cleaning text, and analyzing language, it offers user-friendly tools to streamline workflows.
 Home-page: https://mp.weixin.qq.com/s/lWMkYDWQMjBJNKY2vMYTpw
 Author: Pan Guisheng
 Author-email: panguisheng@sufe.edu.cn
@@ -34,7 +34,7 @@ Key Features:
 4. **Data Storage:** Write and append data to text files, Excel, JSON, and JSON lines.
 5. **File and Folder Processing:** Manage file paths, create directories, move or copy files, and search for files with specific keywords.
 6. **Data Cleaning:** Clean text, handle punctuation, remove stopwords, convert Markdown strings into Python objects, and prepare data for analysis, utilizing valuable corpora and dictionaries such as CET-4/6 vocabulary, BE21 and BNC-COCA word lists.
-7. **NLP:** Perform OCR, word tokenization, lemmatization, POS tagging, NER, dependency parsing, ATE, MDD, WSD, LIWC, and MIP analysis using prepared LLM prompts.
+7. **NLP:** Perform OCR, word tokenization, lemmatization, POS tagging, NER, dependency parsing, ATE, MDD, WSD, LIWC, MIP analysis, and Chinese-English sentence alignment using prepared LLM prompts.
 8. **Math Operations:** Format numbers, convert decimals to percentages, and validate data.
 9. **Visualization:** Process images (e.g., make white pixels transparent, resize images) and manage fonts for rendering text.
 

{PgsFile-0.4.6.dist-info → PgsFile-0.4.8.dist-info}/RECORD

@@ -1,5 +1,5 @@
-PgsFile/PgsFile.py,sha256=lsccmRjB-vHPTeAsGG97CYPjTiTrVnbiiSHCqaGc6sk,168231
-PgsFile/__init__.py,sha256=0yJbrdpue45cageZqejZAsjdWXcEg-Cs2NT7elOF1rQ,3627
+PgsFile/PgsFile.py,sha256=3iyfFE5THgwwz0_MtylUlMn-72gsRaCbUxdm9LcI8nQ,169903
+PgsFile/__init__.py,sha256=mWZ8dfTlzeCfTHFlWyHY3vCwqyM4_YQBGPd6vBoNGso,3674
 PgsFile/Corpora/Idioms/English_Idioms_8774.txt,sha256=qlsP0yI_XGECBRiPZuLkGZpdasc77sWSKexANu7v8_M,175905
 PgsFile/Corpora/Monolingual/Chinese/People's Daily 20130605/Raw/00000000.txt,sha256=SLGGSMSb7Ff1RoBstsTW3yX2wNZpqEUchFNpcI-mrR4,1513
 PgsFile/Corpora/Monolingual/Chinese/People's Daily 20130605/Raw/00000001.txt,sha256=imOa6UoCOIZoPXT4_HNHgCUJtd4FTIdk2FZNHNBgJyg,3372
@@ -2591,8 +2591,9 @@ PgsFile/models/prompts/3. ICTCLAS Prompt.txt,sha256=VFn6N_JViAbyy9NazA8gjX6SGo5m
 PgsFile/models/prompts/4. OCR prompt.txt,sha256=YxUQ2IlE52k0fcBnGsuOHqWAmfiEmIu6iRz5zecQ8dk,260
 PgsFile/models/prompts/5. ATE prompt.txt,sha256=5wu0gGlsV7DI0LruYM3-uAC6brppyYD0IoiFVjMqm5Y,1553
 PgsFile/models/prompts/6. ATE3 prompt.txt,sha256=VnaXpPa6BgZHUcm8PxmP_qgU-8xEoTB3XcBqjwCUy_g,1254
-PgsFile-0.4.6.dist-info/LICENSE,sha256=cE5c-QToSkG1KTUsU8drQXz1vG0EbJWuU4ybHTRb5SE,1138
-PgsFile-0.4.6.dist-info/METADATA,sha256=eraBURV9-2LJqn0sTWNW35L286E1r8DXVX-ZVAHurY4,2994
-PgsFile-0.4.6.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-PgsFile-0.4.6.dist-info/top_level.txt,sha256=028hCfwhF3UpfD6X0rwtWpXI1RKSTeZ1ALwagWaSmX8,8
-PgsFile-0.4.6.dist-info/RECORD,,
+PgsFile/models/prompts/7. SentAlign prompt.txt,sha256=hXpqqC-CAgo8EytkJ0MaLhevLefALazWriY-ew39jxs,1537
+PgsFile-0.4.8.dist-info/LICENSE,sha256=cE5c-QToSkG1KTUsU8drQXz1vG0EbJWuU4ybHTRb5SE,1138
+PgsFile-0.4.8.dist-info/METADATA,sha256=kd3UY3kgL0HMBGe16hqzpLeCjc-A4wdKYGshb7FBecw,3065
+PgsFile-0.4.8.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+PgsFile-0.4.8.dist-info/top_level.txt,sha256=028hCfwhF3UpfD6X0rwtWpXI1RKSTeZ1ALwagWaSmX8,8
+PgsFile-0.4.8.dist-info/RECORD,,