PgsFile 0.4.3__py3-none-any.whl → 0.4.5__py3-none-any.whl

PgsFile/PgsFile.py

@@ -4064,3 +4064,30 @@ def convert_image_to_url(image_path: str) -> str:
 
     return image_url
 
+import ast
+def markdown_to_python_object(data):
+    """
+    If `data` is already a Python object (list, dict, tuple, str, etc.), return it.
+    If it's a Markdown code block, try to parse it into the equivalent Python object.
+    """
+    # If already a Python object (but not a string), return as is
+    if not isinstance(data, str):
+        return data
+
+    # Match Markdown code block (with or without `python`)
+    code_block = re.search(r"```(?:python)?\s*(.*?)\s*```", data, re.DOTALL)
+    if not code_block:
+        # If there's no triple backticks, try parsing the string directly
+        try:
+            return ast.literal_eval(data)
+        except Exception:
+            return data.strip()
+    
+    code_str = code_block.group(1)
+
+    # Try safe parsing
+    try:
+        return ast.literal_eval(code_str)
+    except Exception:
+        return code_str.strip()
+    

PgsFile/__init__.py

@@ -42,6 +42,7 @@ from .PgsFile import replace_chinese_punctuation_with_english
 from .PgsFile import replace_english_punctuation_with_chinese
 from .PgsFile import clean_list, clean_text, clean_text_with_abbreviations, clean_line_with_abbreviations
 from .PgsFile import extract_chinese_punctuation, generate_password, sort_strings_with_embedded_numbers
+from .PgsFile import markdown_to_python_object
 
 # 7. NLP (natural language processing)
 from .PgsFile import strQ2B_raw, strQ2B_words

PgsFile/models/prompts/5. ATE prompt.txt

@@ -0,0 +1,28 @@
+You are an excellent automatic term extraction (ATE) system. Your task is to identify and extract terms from a given text based on a specified domain. Terms should be domain-specific and should not include named entities. I will provide you with the domain and the sentence from which you need to extract the terms. Please follow the output format provided in the examples.
+
+What are the terms in the following text? Terms should not include named entities.
+Output Format: ["list of terms present"]
+If no terms are presented, keep it an empty list: []
+
+EXAMPLES:
+
+Example 1:
+Sentence: "Treatment of anemia in patients with heart disease: a clinical practice guideline from the American College of Physicians."
+Domain: Heart failure
+Output: ["anemia", "patients", "heart disease", "clinical practice guideline", "Physicians"]
+
+Example 2:
+Sentence: "Recommendation 2: ACP recommends against the use of erythropoiesis-stimulating agents in patients with mild to moderate anemia and congestive heart failure or coronary heart disease."
+Domain: Heart failure
+Output: ["erythropoiesis-stimulating agents", "patients", "anemia", "congestive heart failure", "coronary heart disease"]
+
+Example 3:
+Sentence: "Moreover, there is yet to be established a common consensus being used in current assays."
+Domain: Heart failure
+Output: []
+
+Now, please extract the terms from the following sentence:
+
+Sentence: "{sentence}"
+Domain: "{domain}"
+Output: ["list of terms present"]

PgsFile/models/prompts/6. ATE3 prompt.txt

@@ -0,0 +1,18 @@
+Extract terms from the input text and classify each term into one of the following categories based on their domain- and lexicon-specificity:
+•	Specific_Term: Domain-specific and lexicon-specific (known mainly by experts, highly relevant to the domain)
+•	Common_Term: Domain-specific but not lexicon-specific (known by laypersons, still relevant to the domain)
+•	OOD_Term: Lexicon-specific but not domain-specific (known mainly by experts, but not relevant to the domain)
+
+Terms should not include named entities.
+Output Format (a list of python-tuple): [("2-vessel cad", "Specific_Term"), ("aortic valve", "Common_Term"), ("p-value", "OOD_Term")]
+If no terms are presented, keep it an empty list: []
+
+Examples (in the domain of heart failure):
+•	"ejection fraction" → "Specific_Term": laypersons generally do not know what it means, and it is strongly related to the domain.
+•	"heart" → "Common_Term": relevant to the domain and understandable by the general public.
+•	"p-value" → "OOD_Term": expert-level term, but not domain-specific to heart failure.
+
+Now, please extract the terms from the following sentence:
+
+Sentence: "{sentence}"
+Output: ["list of terms present"]

{PgsFile-0.4.3.dist-info → PgsFile-0.4.5.dist-info}/METADATA

@@ -1,7 +1,7 @@
 Metadata-Version: 2.1
 Name: PgsFile
-Version: 0.4.3
-Summary: This module simplifies Python package management, script execution, file handling, web scraping, and multimedia downloads. The module supports (LLM-based) NLP tasks such as OCR, tokenization, lemmatization, POS tagging, NER, keywords extraction, dependency parsing, MDD, WSD, LIWC, and MIP analysis. It also generates word lists, and plots data, aiding literary students. Ideal for scraping data, cleaning text, and analyzing language, it offers user-friendly tools to streamline workflows.
+Version: 0.4.5
+Summary: This module simplifies Python package management, script execution, file handling, web scraping, and multimedia downloads. The module supports (LLM-based) NLP tasks such as OCR, tokenization, lemmatization, POS tagging, NER, ATE, dependency parsing, MDD, WSD, LIWC, and MIP analysis. It also generates word lists, and plots data, aiding literary students. Ideal for scraping data, cleaning text, and analyzing language, it offers user-friendly tools to streamline workflows.
 Home-page: https://mp.weixin.qq.com/s/lWMkYDWQMjBJNKY2vMYTpw
 Author: Pan Guisheng
 Author-email: panguisheng@sufe.edu.cn
@@ -33,8 +33,8 @@ Key Features:
 3. **Data Retrieval:** Extract data from various file formats like text, JSON, TSV, Excel, XML, and HTML (both online and offline).
 4. **Data Storage:** Write and append data to text files, Excel, JSON, and JSON lines.
 5. **File and Folder Processing:** Manage file paths, create directories, move or copy files, and search for files with specific keywords.
-6. **Data Cleaning:** Clean text, handle punctuation, remove stopwords, and prepare data for analysis, utilizing valuable corpora and dictionaries such as CET-4/6 vocabulary, BE21 and BNC-COCA word lists.
-7. **NLP:** Perform OCR, word tokenization, lemmatization, POS tagging, NER, dependency parsing, keywords extraction, MDD, WSD, LIWC, and MIP analysis using prepared LLM prompts.
+6. **Data Cleaning:** Clean text, handle punctuation, remove stopwords, convert Markdown strings into Python objects, and prepare data for analysis, utilizing valuable corpora and dictionaries such as CET-4/6 vocabulary, BE21 and BNC-COCA word lists.
+7. **NLP:** Perform OCR, word tokenization, lemmatization, POS tagging, NER, dependency parsing, ATE, MDD, WSD, LIWC, and MIP analysis using prepared LLM prompts.
 8. **Math Operations:** Format numbers, convert decimals to percentages, and validate data.
 9. **Visualization:** Process images (e.g., make white pixels transparent, resize images) and manage fonts for rendering text.
 

{PgsFile-0.4.3.dist-info → PgsFile-0.4.5.dist-info}/RECORD

@@ -1,5 +1,5 @@
-PgsFile/PgsFile.py,sha256=r7aEpi-ZhmUeLUOLDmEV4g5A8eIsI4tUo4LcQ357rOM,167331
-PgsFile/__init__.py,sha256=H6oQNcJ0MPHV7WQFIlsp8y_1HD_kuxCD18zby4AGZGk,3579
+PgsFile/PgsFile.py,sha256=lsccmRjB-vHPTeAsGG97CYPjTiTrVnbiiSHCqaGc6sk,168231
+PgsFile/__init__.py,sha256=0yJbrdpue45cageZqejZAsjdWXcEg-Cs2NT7elOF1rQ,3627
 PgsFile/Corpora/Idioms/English_Idioms_8774.txt,sha256=qlsP0yI_XGECBRiPZuLkGZpdasc77sWSKexANu7v8_M,175905
 PgsFile/Corpora/Monolingual/Chinese/People's Daily 20130605/Raw/00000000.txt,sha256=SLGGSMSb7Ff1RoBstsTW3yX2wNZpqEUchFNpcI-mrR4,1513
 PgsFile/Corpora/Monolingual/Chinese/People's Daily 20130605/Raw/00000001.txt,sha256=imOa6UoCOIZoPXT4_HNHgCUJtd4FTIdk2FZNHNBgJyg,3372
@@ -2589,8 +2589,10 @@ PgsFile/models/prompts/1. MIP prompt.txt,sha256=4lHlHmleayRytqr1n9jtt6vn1rQvyf4B
 PgsFile/models/prompts/2. WSD prompt.txt,sha256=o-ZFtCRUCDrXgm040WTQch9v2Y_r2SIlrZaquilJjgQ,2348
 PgsFile/models/prompts/3. ICTCLAS Prompt.txt,sha256=VFn6N_JViAbyy9NazA8gjX6SGo5mgBcZOf95aC9JB84,592
 PgsFile/models/prompts/4. OCR prompt.txt,sha256=YxUQ2IlE52k0fcBnGsuOHqWAmfiEmIu6iRz5zecQ8dk,260
-PgsFile-0.4.3.dist-info/LICENSE,sha256=cE5c-QToSkG1KTUsU8drQXz1vG0EbJWuU4ybHTRb5SE,1138
-PgsFile-0.4.3.dist-info/METADATA,sha256=51z08PbAHn17jX4KOA7U3Nz_7_id-RtX9w0eLUYXS5o,2980
-PgsFile-0.4.3.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-PgsFile-0.4.3.dist-info/top_level.txt,sha256=028hCfwhF3UpfD6X0rwtWpXI1RKSTeZ1ALwagWaSmX8,8
-PgsFile-0.4.3.dist-info/RECORD,,
+PgsFile/models/prompts/5. ATE prompt.txt,sha256=ZJo9BhbbUf7CVXi2Gb5DAsV_2PGgzly2I7ze0grCo2k,1486
+PgsFile/models/prompts/6. ATE3 prompt.txt,sha256=BcefVd-RM1_IjSH1t9JfRUz6qxIoYrDxORLlysT3H70,1209
+PgsFile-0.4.5.dist-info/LICENSE,sha256=cE5c-QToSkG1KTUsU8drQXz1vG0EbJWuU4ybHTRb5SE,1138
+PgsFile-0.4.5.dist-info/METADATA,sha256=trHMmN-jQGE_NxXW35ZdVyOf7XC_Q9xT5v4vMkk48rw,2994
+PgsFile-0.4.5.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+PgsFile-0.4.5.dist-info/top_level.txt,sha256=028hCfwhF3UpfD6X0rwtWpXI1RKSTeZ1ALwagWaSmX8,8
+PgsFile-0.4.5.dist-info/RECORD,,