PgsFile 0.3.6__py3-none-any.whl → 0.3.8__py3-none-any.whl

PgsFile/PgsFile.py

@@ -3667,3 +3667,132 @@ def timeit(func):
         return result
     return wrapper
 
+def file_to_list_of_dicts(input_path, output_path):
+    """
+    Convert a CSV or XLSX file into a JSON file, where each line in the JSON file is a dictionary representing a row from the input file.
+    The keys of each dictionary are formatted as "column1, column2, column3, etc."
+
+    Args:
+        input_path (str): Path to the CSV or XLSX file.
+        output_path (str): Path to the output JSON file.
+
+    Returns:
+        None: The function writes the dictionaries to a JSON file specified by output_path.
+    """
+    # Determine the file type based on the file extension
+    import pandas as pd
+    if input_path.endswith('.csv'):
+        df = pd.read_csv(input_path)
+    elif input_path.endswith('.xlsx'):
+        df = pd.read_excel(input_path)
+    else:
+        raise ValueError("Unsupported file format. Please provide a CSV or XLSX file.")
+
+    # Use default keys "col1, col2, etc."
+    total = len(list(df.iterrows()))
+    for index, row in df.iterrows():
+        row_dict = {f"col{i+1}": value for i, value in enumerate(row)}
+        append_dict_to_json(output_path, row_dict)
+        print(f'Converting {decimal_to_percent(index/total)}')
+
+    print("Conversion complete!")
+    
+
+import liwc
+import json
+def perform_liwc_en(dic_path, file_path, output_excel_path):
+    '''
+    Parameters
+    ----------
+    dic_path : str
+        Path to the LIWC dictionary file.
+    file_path : str
+        Path to the raw text file.
+    output_excel_path : str
+        Path to the output Excel file.
+    '''
+    parse, category_names = liwc.load_token_parser(dic_path)
+    test = get_data_text(file_path)
+    test_tokens = [w.lower() for w in word_tokenize2(test)]
+    my_word_list = dict(word_list(test_tokens))
+    corpus_scale = sum(my_word_list.values())
+
+    labels = {'function': '1', 'pronoun': '2', 'ppron': '3', 'i': '4', 'we': '5', 'you': '6', 'shehe': '7', 'they': '8', 'ipron': '9', 'article': '10', 'prep': '11', 'auxverb': '12', 'adverb': '13', 'conj': '14', 'negate': '15', 'verb': '20', 'adj': '21', 'compare': '22', 'interrog': '23', 'number': '24', 'quant': '25', 'affect': '30', 'posemo': '31', 'negemo': '32', 'anx': '33', 'anger': '34', 'sad': '35', 'social': '40', 'family': '41', 'friend': '42', 'female': '43', 'male': '44', 'cogproc': '50', 'insight': '51', 'cause': '52', 'discrep': '53', 'tentat': '54', 'certain': '55', 'differ': '56', 'percept': '60', 'see': '61', 'hear': '62', 'feel': '63', 'bio': '70', 'body': '71', 'health': '72', 'sexual': '73', 'ingest': '74', 'drives': '80', 'affiliation': '81', 'achiev': '82', 'power': '83', 'reward': '84', 'risk': '85', 'focuspast': '90', 'focuspresent': '91', 'focusfuture': '92', 'relativ': '100', 'motion': '101', 'space': '102', 'time': '103', 'work': '110', 'leisure': '111', 'home': '112', 'money': '113', 'relig': '114', 'death': '115', 'informal': '120', 'swear': '121', 'netspeak': '122', 'assent': '123', 'nonflu': '124', 'filler': '125', 'punctuation':'126'}
+
+    # 第一列：类别标签
+    # 第二列：某类别出现的词种数
+    # 第三列：某类别出现的词次数
+    # 第四列：考察文本语料库的总词次数
+    # 第五列：覆盖率
+    # 第六列：例词，包含某个词的频次降序排列
+    
+    def get_category_info(x):
+        category_words = []
+        category_words_freq = 0
+        for w in my_word_list:
+            categories = parse(w)
+            if x in categories:
+                category_words.append([w, my_word_list[w]])
+                category_words_freq += my_word_list[w]
+
+        final = sorted(category_words, key=lambda x: x[1], reverse=True)
+        json_string = json.dumps(final)
+        return [len(category_words), category_words_freq, corpus_scale, decimal_to_percent(category_words_freq / corpus_scale), json_string]
+
+    data = []
+    labels_list = list(labels.keys())
+    for i in labels_list:
+        rs = get_category_info(i)
+        data.append([i] + rs)
+
+    import pandas as pd
+    df = pd.DataFrame(data, columns=[u'类别', u'出现词种数', u'出现词次', u'总词次', u'覆盖率', u'例词'])
+    df.to_excel(output_excel_path, 'sheet1', index=False)
+
+
+def perform_liwc_zh(dic_path, file_path, output_excel_path):
+    '''
+    Parameters
+    ----------
+    dic_path : str
+        Path to the LIWC dictionary json file.
+    file_path : str
+        Path to the raw text file.
+    output_excel_path : str
+        Path to the output Excel file.
+    '''        
+    
+    f=open(dic_path,"r")
+    dicx=json.load(f)    
+    
+    test = get_data_text(file_path)
+    test_tokens = word_tokenize(test)
+    my_word_list=dict(word_list(test_tokens))
+    corpus_scale=sum(my_word_list.values())
+    
+    labels=['Entry:词条','function:功能词','pronoun:代名词','ppron:特定人称代名词','i:第一人称单数代名词','we:第一人称复数代名词','you:第二人称代名词','shehe:第三人称单数代名词','they:第三人称复数代名词','youpl:第二人称复数代名词','ipron:非特定人称代名词','prep:介系词','auxverb:助动词','adverb:副词','conj:连接词','negate:否定词','quanunit:量词','prepend:后置词','specart:特指定词','particle:小品词','modal_pa:语气词','general_pa','compare:比较词','interrog:疑问词','number:数字','quant:概数词','affect:情感历程词','posemo:正向情绪词','negemo:负向情绪词','anx:焦虑词','anger:生气词','sad:悲伤词','social:社会历程词','family:家族词','friend:朋友词','female:女性词','male:男性词','cogproc','insight:洞察词','cause:因果词','discrep:差距词','tentat:暂定词','certain:确切词','differ','percept:感知历程词','see:视觉词','hear:听觉词','feel:感觉词','bio:生理历程词','body:身体词','health:健康词','sexual:性词','ingest:摄食词','drives','affiliation','achieve:成就词','power','reward','risk','tensem:时态标定词','focuspast:过去时态标定词','focuspresent:现在时态标定词','focusfuture:未来时态标定词','progm:延续时态标定词','relativ:相对词','motion:移动词','space:空间词','time:时间词','work:工作词','leisure:休闲词','home:家庭词','money:金钱词','relig:宗教词','death:死亡词','informal','swear:脏话','netspeak:网络用语','assent:应和词','nonflu:停顿赘词','filler:填充赘词','punctuation:标点符号']
+    # target_index=labels.index("swear") #75
+    
+    def get_category_info(x):
+        category_words=[]
+        category_words_indicx=[]
+        category_words_freq=0
+        for w in dicx:
+            if dicx[w][x]=="1":
+                category_words_indicx.append(w)
+                if w in my_word_list:
+                    category_words_freq+=my_word_list[w]
+                    category_words.append([w,my_word_list[w]])
+        final=sorted(category_words,key=lambda x: x[1],reverse=True)
+        json_string = json.dumps(final, ensure_ascii=False)
+        return [len(category_words),decimal_to_percent(len(category_words)/len(category_words_indicx)),category_words_freq,corpus_scale, decimal_to_percent(category_words_freq/corpus_scale), json_string]
+    
+    data=[]
+    for i in labels[1::]:
+        rs=get_category_info(labels[1::].index(i)) 
+        data.append([i]+rs)
+    
+    
+    import pandas as pd
+    df = pd.DataFrame(data,columns=[u'类别', u'出现词种数', u'占词表百分比', u'出现词次', u'总词次', u'覆盖率', u'例词'])
+    df.to_excel(output_excel_path,'sheet1',index=False)

PgsFile/__init__.py

@@ -17,7 +17,7 @@ from .PgsFile import get_data_table_url, get_data_table_html_string
 from .PgsFile import mhtml2html
 
 # 4. Text data storage
-from .PgsFile import write_to_txt, write_to_excel, write_to_json, write_to_json_lines, append_dict_to_json, save_dict_to_excel
+from .PgsFile import write_to_txt, write_to_excel, write_to_json, write_to_json_lines, append_dict_to_json, save_dict_to_excel, file_to_list_of_dicts
 from .PgsFile import write_to_excel_normal
 
 # 5. File/folder process
@@ -45,7 +45,7 @@ from .PgsFile import extract_chinese_punctuation, generate_password, sort_string
 
 # 7. NLP (natural language processing)
 from .PgsFile import strQ2B_raw, strQ2B_words
-from .PgsFile import ngrams, bigrams, trigrams, everygrams, compute_similarity
+from .PgsFile import ngrams, bigrams, trigrams, everygrams, compute_similarity, perform_liwc_en, perform_liwc_zh
 from .PgsFile import word_list, batch_word_list
 from .PgsFile import cs, cs1, sent_tokenize, word_tokenize, word_tokenize2
 from .PgsFile import word_lemmatize, word_POS, word_NER

{PgsFile-0.3.6.dist-info → PgsFile-0.3.8.dist-info}/METADATA

@@ -1,7 +1,7 @@
 Metadata-Version: 2.1
 Name: PgsFile
-Version: 0.3.6
-Summary: This module simplifies Python package management, script execution, file handling, web scraping, and multimedia downloads. The module supports LLM-based NLP tasks such as tokenization, lemmatization, POS tagging, NER, dependency parsing, MDD, WSD, and MIP analysis. It also generates word lists and plots data, aiding literary students. Ideal for scraping data, cleaning text, and analyzing language, it offers user-friendly tools to streamline workflows.
+Version: 0.3.8
+Summary: This module simplifies Python package management, script execution, file handling, web scraping, and multimedia downloads. The module supports LLM-based NLP tasks such as tokenization, lemmatization, POS tagging, NER, dependency parsing, MDD, WSD, LIWC, and MIP analysis. It also generates word lists and plots data, aiding literary students. Ideal for scraping data, cleaning text, and analyzing language, it offers user-friendly tools to streamline workflows.
 Home-page: https://mp.weixin.qq.com/s/12-KVLfaPszoZkCxuRd-nQ?token=1589547443&lang=zh_CN
 Author: Pan Guisheng
 Author-email: 895284504@qq.com
@@ -23,6 +23,7 @@ Requires-Dist: pimht
 Requires-Dist: pysbd
 Requires-Dist: nlpir-python
 Requires-Dist: pillow
+Requires-Dist: liwc
 
 Purpose: This module is designed to make complex tasks accessible and convenient, even for beginners. By providing a unified set of tools, it simplifies the workflow for data collection, processing, and analysis. Whether you're scraping data from the web, cleaning text, or performing LLM-based NLP tasks, this module ensures you can focus on your research without getting bogged down by technical challenges.
 
@@ -33,7 +34,7 @@ Key Features:
 4. **Data Storage:** Write and append data to text files, Excel, JSON, and JSON lines.
 5. **File and Folder Processing:** Manage file paths, create directories, move or copy files, and search for files with specific keywords.
 6. **Data Cleaning:** Clean text, handle punctuation, remove stopwords, and prepare data for analysis, utilizing valuable corpora and dictionaries such as CET-4/6 vocabulary and BNC-COCA word lists.
-7. **NLP:** Perform word tokenization, lemmatization, POS tagging, NER, dependency parsing, MDD, WSD, and MIP analysis using prepared LLM prompts.
+7. **NLP:** Perform word tokenization, lemmatization, POS tagging, NER, dependency parsing, MDD, WSD, LIWC, and MIP analysis using prepared LLM prompts.
 8. **Math Operations:** Format numbers, convert decimals to percentages, and validate data.
 9. **Visualization:** Process images (e.g., make white pixels transparent) and manage fonts for rendering text.
 

{PgsFile-0.3.6.dist-info → PgsFile-0.3.8.dist-info}/RECORD

@@ -1,5 +1,5 @@
-PgsFile/PgsFile.py,sha256=52Uxj1gii1F1J9rvWnR1cFffIumeqBDsJRaN_uLoZUg,149704
-PgsFile/__init__.py,sha256=9vTeHtnxXaf_Qo36pes9o5_MU_M5M7MSMUddDWvkoDA,3408
+PgsFile/PgsFile.py,sha256=MgXheIxpcTcq6vf7t-Ak6JwVmoamS9ouDuOUDeaYpUY,157289
+PgsFile/__init__.py,sha256=c4v2XTQjk5_ato5qeLzrS6mYm7tHR_V-Pb0QFgJHXJg,3465
 PgsFile/Corpora/Idioms/English_Idioms_8774.txt,sha256=qlsP0yI_XGECBRiPZuLkGZpdasc77sWSKexANu7v8_M,175905
 PgsFile/Corpora/Monolingual/Chinese/People's Daily 20130605/Raw/00000000.txt,sha256=SLGGSMSb7Ff1RoBstsTW3yX2wNZpqEUchFNpcI-mrR4,1513
 PgsFile/Corpora/Monolingual/Chinese/People's Daily 20130605/Raw/00000001.txt,sha256=imOa6UoCOIZoPXT4_HNHgCUJtd4FTIdk2FZNHNBgJyg,3372
@@ -2585,8 +2585,8 @@ PgsFile/models/fonts/博洋行书3500.TTF,sha256=VrgeHr8cgOL6JD05QyuD9ZSyw4J2aIV
 PgsFile/models/fonts/陆柬之行书字体.ttf,sha256=Zpd4Z7E9w-Qy74yklXHk4vM7HOtHuQgllvygxZZ1Hvs,1247288
 PgsFile/models/prompts/1. MIP prompt.txt,sha256=4lHlHmleayRytqr1n9jtt6vn1rQvyf4BKeThpbwI8o8,1638
 PgsFile/models/prompts/2. WSD prompt.txt,sha256=o-ZFtCRUCDrXgm040WTQch9v2Y_r2SIlrZaquilJjgQ,2348
-PgsFile-0.3.6.dist-info/LICENSE,sha256=cE5c-QToSkG1KTUsU8drQXz1vG0EbJWuU4ybHTRb5SE,1138
-PgsFile-0.3.6.dist-info/METADATA,sha256=HkRDJ8CCZoGrkKIuwlSpju61tF5bdn-2Hbrwt4B6zd8,2892
-PgsFile-0.3.6.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-PgsFile-0.3.6.dist-info/top_level.txt,sha256=028hCfwhF3UpfD6X0rwtWpXI1RKSTeZ1ALwagWaSmX8,8
-PgsFile-0.3.6.dist-info/RECORD,,
+PgsFile-0.3.8.dist-info/LICENSE,sha256=cE5c-QToSkG1KTUsU8drQXz1vG0EbJWuU4ybHTRb5SE,1138
+PgsFile-0.3.8.dist-info/METADATA,sha256=TmXJtWuYSG4YSbsPylK2gKNip0QtDmd7NCRG84zb6Lk,2925
+PgsFile-0.3.8.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+PgsFile-0.3.8.dist-info/top_level.txt,sha256=028hCfwhF3UpfD6X0rwtWpXI1RKSTeZ1ALwagWaSmX8,8
+PgsFile-0.3.8.dist-info/RECORD,,