PgsFile 0.2.3__py3-none-any.whl → 0.2.4__py3-none-any.whl

PgsFile/PgsFile.py

@@ -103,7 +103,7 @@ def get_data_text(path):
     else:
         return None
 
-def get_data_lines(path):
+def get_data_lines(path, no_line_breaks=False):
     '''
     Parameters
     ----------
@@ -133,7 +133,10 @@ def get_data_lines(path):
     # Read the entire file using the detected encoding
     if encoding:
         with open(path, 'r', encoding=encoding, errors="ignore") as f:
-            lines = [l.strip() for l in f.readlines() if len(l.strip()) != 0]
+            if no_line_breaks is False:
+                lines = [l.strip() for l in f.readlines() if len(l.strip()) != 0]
+            else:
+                lines = f.readlines()
             return lines
     else:
         return None
@@ -197,15 +200,15 @@ def get_data_excel(excel_path,column_id,sheet_name=None):
     inter=df.iloc[0:,column_id] #提取第二列所有行  
     return list(inter)
 
-def write_to_excel(excel_path,dic_of_list,sheet_name=None,index=None):
+def write_to_excel(excel_path, data, sheet_name=None, index=None):
     '''
     Parameters
     ----------
     excel_path : TYPE
         DESCRIPTION. results.xlsx
         
-    dic_of_list : TYPE
-        DESCRIPTION. {"col":["a","b","c","d"],"freq":[1,2,3,4]}
+    data : TYPE, dict
+        DESCRIPTION. data = {'翻译': 24, '教学': 8, '数智': 6, '时代': 6, '财经': 6, '新': 4}
         
     sheet_name : TYPE, optional
         DESCRIPTION. The default is None.
@@ -227,6 +230,10 @@ def write_to_excel(excel_path,dic_of_list,sheet_name=None,index=None):
         index=False
     else:
         index=True        
+
+    col = list(data.keys())
+    freq = list(data.values())
+    dic_of_list={"items": col, "counts": freq}
         
     df=pd.DataFrame(dic_of_list)
     df.style.to_excel(excel_path, sheet_name=sheet_name,startcol=0, index=index)
@@ -471,6 +478,18 @@ def get_directory_tree_with_meta(start_path, indent='', show_meta=False, max_dir
         print(f"{indent}└── ... (and {remaining_directories} more directories)")
         # current_level=-1 will show all folders' info.
 
+def get_full_path(*path_components):
+    """
+    Combines multiple path components into a single, full path using os.path.join.
+
+    Args:
+        *path_components: Variable number of path components (strings).
+
+    Returns:
+        str: The combined full path.
+    """
+    return os.path.join(*path_components)
+
 def get_subfolder_path(parent_folder, subfolder_name):
     import os
     subfolder_name=subfolder_name.strip()
@@ -553,7 +572,6 @@ def batch_word_list(input_root):
     sorted_words=sorted(word_counts.items(), key=lambda x: x[1], reverse=True)
     return sorted_words 
 
-	
 def clean_list(meta):
     """
     Parameters
@@ -576,7 +594,6 @@ def clean_list(meta):
 
 yhd=["Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36','Mozilla/4.0 (compatible; MSIE 6.0; ) Opera/UCWEB7.0.2.37/28/999','Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)','Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)','Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)','Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser)','Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)','Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)','Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)','Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)','Mozilla/5.0 (BlackBerry; U; BlackBerry 9800; en) AppleWebKit/534.1+ (KHTML, like Gecko) Version/6.0.0.337 Mobile Safari/534.1+','Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0','Mozilla/5.0 (compatible; MSIE 9.0; Windows Phone OS 7.5; Trident/5.0; IEMobile/9.0; HTC; Titan)','Mozilla/5.0 (iPad; U; CPU OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5','Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5','Mozilla/5.0 (iPod; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5','Mozilla/5.0 (Linux; U; Android 2.3.7; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1','Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13','Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1','Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1','Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50','Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6','NOKIA5700/ UCWEB7.0.2.37/28/999','Openwave/ UCWEB7.0.2.37/28/999','Opera/9.80 (Android 2.3.4; Linux; Opera Mobi/build-1107180945; U; en-GB) Presto/2.8.149 Version/11.10','Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11','Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11','UCWEB7.0.2.37/28/999']
 
-
 def source_path(relative_path):
     import sys,os
     if getattr(sys, 'frozen', False): 
@@ -590,7 +607,6 @@ def next_folder_names(folder):
     folder_namelist=next(os.walk(folder))[1]
     return folder_namelist
 
-
 def remove_empty_txts(folder_path):
     import os
     files=FilePath(folder_path)
@@ -797,7 +813,6 @@ def cs1(text):
         sentences=sentences
     return sentences
 
-                
 def word_tokenize(text, pos_tagged=False):
     '''
     Parameters
@@ -838,7 +853,7 @@ def word_tokenize(text, pos_tagged=False):
         else:
             try:
                 if "Can not open" in str(err):
-                    user_folder=get_library_location("PgsFile")+"/PgsFile/Corpora/Stopwords"
+                    user_folder=get_library_location("PgsFile")+"/PgsFile/models"
                     destination_folder=get_library_location("nlpir-python")+"/nlpir/Data"
                     source_file=find_user_files_in_upper_folder(user_folder, "NLPIR")[0]
                     copy_file(source_file, destination_folder)  
@@ -850,6 +865,296 @@ def word_tokenize(text, pos_tagged=False):
 
     return words  
 
+import re
+from abc import ABC, abstractmethod
+from typing import Iterator, List, Tuple
+class TokenizerI(ABC):
+    """
+    A processing interface for tokenizing a string.
+    Subclasses must define ``tokenize()`` or ``tokenize_sents()`` (or both).
+    """
+
+    @abstractmethod
+    def tokenize(self, s: str) -> List[str]:
+        """
+        Return a tokenized copy of *s*.
+
+        :rtype: List[str]
+        """
+        if overridden(self.tokenize_sents):
+            return self.tokenize_sents([s])[0]
+
+    def span_tokenize(self, s: str) -> Iterator[Tuple[int, int]]:
+        """
+        Identify the tokens using integer offsets ``(start_i, end_i)``,
+        where ``s[start_i:end_i]`` is the corresponding token.
+
+        :rtype: Iterator[Tuple[int, int]]
+        """
+        raise NotImplementedError()
+
+    def tokenize_sents(self, strings: List[str]) -> List[List[str]]:
+        """
+        Apply ``self.tokenize()`` to each element of ``strings``.  I.e.:
+
+            return [self.tokenize(s) for s in strings]
+
+        :rtype: List[List[str]]
+        """
+        return [self.tokenize(s) for s in strings]
+
+    def span_tokenize_sents(
+        self, strings: List[str]
+    ) -> Iterator[List[Tuple[int, int]]]:
+        """
+        Apply ``self.span_tokenize()`` to each element of ``strings``.  I.e.:
+
+            return [self.span_tokenize(s) for s in strings]
+
+        :yield: List[Tuple[int, int]]
+        """
+        for s in strings:
+            yield list(self.span_tokenize(s))
+
+class MacIntyreContractions:
+    """
+    List of contractions adapted from Robert MacIntyre's tokenizer.
+    """
+
+    CONTRACTIONS2 = [
+        r"(?i)\b(can)(?#X)(not)\b",
+        r"(?i)\b(d)(?#X)('ye)\b",
+        r"(?i)\b(gim)(?#X)(me)\b",
+        r"(?i)\b(gon)(?#X)(na)\b",
+        r"(?i)\b(got)(?#X)(ta)\b",
+        r"(?i)\b(lem)(?#X)(me)\b",
+        r"(?i)\b(more)(?#X)('n)\b",
+        r"(?i)\b(wan)(?#X)(na)(?=\s)",
+    ]
+    CONTRACTIONS3 = [r"(?i) ('t)(?#X)(is)\b", r"(?i) ('t)(?#X)(was)\b"]
+    CONTRACTIONS4 = [r"(?i)\b(whad)(dd)(ya)\b", r"(?i)\b(wha)(t)(cha)\b"]
+            
+class NLTKWordTokenizer(TokenizerI):
+    """
+    The NLTK tokenizer that has improved upon the TreebankWordTokenizer.
+
+    This is the method that is invoked by ``word_tokenize()``.  It assumes that the
+    text has already been segmented into sentences, e.g. using ``sent_tokenize()``.
+
+    The tokenizer is "destructive" such that the regexes applied will munge the
+    input string to a state beyond re-construction. It is possible to apply
+    `TreebankWordDetokenizer.detokenize` to the tokenized outputs of
+    `NLTKDestructiveWordTokenizer.tokenize` but there's no guarantees to
+    revert to the original string.
+    """
+
+    # Starting quotes.
+    STARTING_QUOTES = [
+        (re.compile("([«“‘„]|[`]+)", re.U), r" \1 "),
+        (re.compile(r"^\""), r"``"),
+        (re.compile(r"(``)"), r" \1 "),
+        (re.compile(r"([ \(\[{<])(\"|\'{2})"), r"\1 `` "),
+        (re.compile(r"(?i)(\')(?!re|ve|ll|m|t|s|d|n)(\w)\b", re.U), r"\1 \2"),
+    ]
+
+    # Ending quotes.
+    ENDING_QUOTES = [
+        (re.compile("([»”’])", re.U), r" \1 "),
+        (re.compile(r"''"), " '' "),
+        (re.compile(r'"'), " '' "),
+        (re.compile(r"\s+"), " "),
+        (re.compile(r"([^' ])('[sS]|'[mM]|'[dD]|') "), r"\1 \2 "),
+        (re.compile(r"([^' ])('ll|'LL|'re|'RE|'ve|'VE|n't|N'T) "), r"\1 \2 "),
+    ]
+
+    # For improvements for starting/closing quotes from TreebankWordTokenizer,
+    # see discussion on https://github.com/nltk/nltk/pull/1437
+    # Adding to TreebankWordTokenizer, nltk.word_tokenize now splits on
+    # - chevron quotes u'\xab' and u'\xbb'
+    # - unicode quotes u'\u2018', u'\u2019', u'\u201c' and u'\u201d'
+    # See https://github.com/nltk/nltk/issues/1995#issuecomment-376741608
+    # Also, behavior of splitting on clitics now follows Stanford CoreNLP
+    # - clitics covered (?!re|ve|ll|m|t|s|d)(\w)\b
+
+    # Punctuation.
+    PUNCTUATION = [
+        (re.compile(r'([^\.])(\.)([\]\)}>"\'' "»”’ " r"]*)\s*$", re.U), r"\1 \2 \3 "),
+        (re.compile(r"([:,])([^\d])"), r" \1 \2"),
+        (re.compile(r"([:,])$"), r" \1 "),
+        (
+            re.compile(r"\.{2,}", re.U),
+            r" \g<0> ",
+        ),  # See https://github.com/nltk/nltk/pull/2322
+        (re.compile(r"[;@#$%&]"), r" \g<0> "),
+        (
+            re.compile(r'([^\.])(\.)([\]\)}>"\']*)\s*$'),
+            r"\1 \2\3 ",
+        ),  # Handles the final period.
+        (re.compile(r"[?!]"), r" \g<0> "),
+        (re.compile(r"([^'])' "), r"\1 ' "),
+        (
+            re.compile(r"[*]", re.U),
+            r" \g<0> ",
+        ),  # See https://github.com/nltk/nltk/pull/2322
+    ]
+
+    # Pads parentheses
+    PARENS_BRACKETS = (re.compile(r"[\]\[\(\)\{\}\<\>]"), r" \g<0> ")
+
+    # Optionally: Convert parentheses, brackets and converts them to PTB symbols.
+    CONVERT_PARENTHESES = [
+        (re.compile(r"\("), "-LRB-"),
+        (re.compile(r"\)"), "-RRB-"),
+        (re.compile(r"\["), "-LSB-"),
+        (re.compile(r"\]"), "-RSB-"),
+        (re.compile(r"\{"), "-LCB-"),
+        (re.compile(r"\}"), "-RCB-"),
+    ]
+
+    DOUBLE_DASHES = (re.compile(r"--"), r" -- ")
+
+    # List of contractions adapted from Robert MacIntyre's tokenizer.
+    _contractions = MacIntyreContractions()
+    CONTRACTIONS2 = list(map(re.compile, _contractions.CONTRACTIONS2))
+    CONTRACTIONS3 = list(map(re.compile, _contractions.CONTRACTIONS3))
+
+    def tokenize(
+        self, text: str, convert_parentheses: bool = False, return_str: bool = False
+    ) -> List[str]:
+        r"""Return a tokenized copy of `text`.
+
+        >>> from nltk.tokenize import NLTKWordTokenizer
+        >>> s = '''Good muffins cost $3.88 (roughly 3,36 euros)\nin New York.  Please buy me\ntwo of them.\nThanks.'''
+        >>> NLTKWordTokenizer().tokenize(s) # doctest: +NORMALIZE_WHITESPACE
+        ['Good', 'muffins', 'cost', '$', '3.88', '(', 'roughly', '3,36',
+        'euros', ')', 'in', 'New', 'York.', 'Please', 'buy', 'me', 'two',
+        'of', 'them.', 'Thanks', '.']
+        >>> NLTKWordTokenizer().tokenize(s, convert_parentheses=True) # doctest: +NORMALIZE_WHITESPACE
+        ['Good', 'muffins', 'cost', '$', '3.88', '-LRB-', 'roughly', '3,36',
+        'euros', '-RRB-', 'in', 'New', 'York.', 'Please', 'buy', 'me', 'two',
+        'of', 'them.', 'Thanks', '.']
+
+
+        :param text: A string with a sentence or sentences.
+        :type text: str
+        :param convert_parentheses: if True, replace parentheses to PTB symbols,
+            e.g. `(` to `-LRB-`. Defaults to False.
+        :type convert_parentheses: bool, optional
+        :param return_str: If True, return tokens as space-separated string,
+            defaults to False.
+        :type return_str: bool, optional
+        :return: List of tokens from `text`.
+        :rtype: List[str]
+        """
+        if return_str:
+            warnings.warn(
+                "Parameter 'return_str' has been deprecated and should no "
+                "longer be used.",
+                category=DeprecationWarning,
+                stacklevel=2,
+            )
+
+        for regexp, substitution in self.STARTING_QUOTES:
+            text = regexp.sub(substitution, text)
+
+        for regexp, substitution in self.PUNCTUATION:
+            text = regexp.sub(substitution, text)
+
+        # Handles parentheses.
+        regexp, substitution = self.PARENS_BRACKETS
+        text = regexp.sub(substitution, text)
+        # Optionally convert parentheses
+        if convert_parentheses:
+            for regexp, substitution in self.CONVERT_PARENTHESES:
+                text = regexp.sub(substitution, text)
+
+        # Handles double dash.
+        regexp, substitution = self.DOUBLE_DASHES
+        text = regexp.sub(substitution, text)
+
+        # add extra space to make things easier
+        text = " " + text + " "
+
+        for regexp, substitution in self.ENDING_QUOTES:
+            text = regexp.sub(substitution, text)
+
+        for regexp in self.CONTRACTIONS2:
+            text = regexp.sub(r" \1 \2 ", text)
+        for regexp in self.CONTRACTIONS3:
+            text = regexp.sub(r" \1 \2 ", text)
+
+        # We are not using CONTRACTIONS4 since
+        # they are also commented out in the SED scripts
+        # for regexp in self._contractions.CONTRACTIONS4:
+        #     text = regexp.sub(r' \1 \2 \3 ', text)
+
+        return text.split()
+
+    def span_tokenize(self, text: str) -> Iterator[Tuple[int, int]]:
+        r"""
+        Returns the spans of the tokens in ``text``.
+        Uses the post-hoc nltk.tokens.align_tokens to return the offset spans.
+
+            >>> from nltk.tokenize import NLTKWordTokenizer
+            >>> s = '''Good muffins cost $3.88\nin New (York).  Please (buy) me\ntwo of them.\n(Thanks).'''
+            >>> expected = [(0, 4), (5, 12), (13, 17), (18, 19), (19, 23),
+            ... (24, 26), (27, 30), (31, 32), (32, 36), (36, 37), (37, 38),
+            ... (40, 46), (47, 48), (48, 51), (51, 52), (53, 55), (56, 59),
+            ... (60, 62), (63, 68), (69, 70), (70, 76), (76, 77), (77, 78)]
+            >>> list(NLTKWordTokenizer().span_tokenize(s)) == expected
+            True
+            >>> expected = ['Good', 'muffins', 'cost', '$', '3.88', 'in',
+            ... 'New', '(', 'York', ')', '.', 'Please', '(', 'buy', ')',
+            ... 'me', 'two', 'of', 'them.', '(', 'Thanks', ')', '.']
+            >>> [s[start:end] for start, end in NLTKWordTokenizer().span_tokenize(s)] == expected
+            True
+
+        :param text: A string with a sentence or sentences.
+        :type text: str
+        :yield: Tuple[int, int]
+        """
+        raw_tokens = self.tokenize(text)
+
+        # Convert converted quotes back to original double quotes
+        # Do this only if original text contains double quote(s) or double
+        # single-quotes (because '' might be transformed to `` if it is
+        # treated as starting quotes).
+        if ('"' in text) or ("''" in text):
+            # Find double quotes and converted quotes
+            matched = [m.group() for m in re.finditer(r"``|'{2}|\"", text)]
+
+            # Replace converted quotes back to double quotes
+            tokens = [
+                matched.pop(0) if tok in ['"', "``", "''"] else tok
+                for tok in raw_tokens
+            ]
+        else:
+            tokens = raw_tokens
+
+        yield from align_tokens(tokens, text)
+
+# Standard word tokenizer.
+_treebank_word_tokenizer = NLTKWordTokenizer()
+def word_tokenize2(text, preserve_line=False):
+    """
+    Return a tokenized copy of *text*,
+    using NLTK's recommended word tokenizer
+    (currently an improved :class:`.TreebankWordTokenizer`
+    along with :class:`.PunktSentenceTokenizer`
+    for the specified language).
+
+    :param text: text to split into words
+    :type text: str
+    :param language: the model name in the Punkt corpus
+    :type language: str
+    :param preserve_line: A flag to decide whether to sentence tokenize the text or not.
+    :type preserve_line: bool
+    """
+    sentences = [text] if preserve_line else sent_tokenize(text)
+    return [
+        token for sent in sentences for token in _treebank_word_tokenizer.tokenize(sent)
+    ]
+        
 def pad_sequence(
     sequence,
     n,
@@ -889,9 +1194,7 @@ def pad_sequence(
         sequence=chain(sequence, (right_pad_symbol,) * (n - 1))
     return sequence
 
-
 # add a flag to pad the sequence so we get peripheral ngrams?
-
 def ngrams(
     sequence,
     n,
@@ -954,7 +1257,6 @@ def ngrams(
         yield tuple(history)
         del history[0]
 
-
 def bigrams(sequence, **kwargs):
     """
     Return the bigrams generated from a sequence of items, as an iterator.
@@ -974,7 +1276,6 @@ def bigrams(sequence, **kwargs):
     for item in ngrams(sequence, 2, **kwargs):
         yield item
 
-
 def trigrams(sequence, **kwargs):
     """
     Return the trigrams generated from a sequence of items, as an iterator.
@@ -994,7 +1295,6 @@ def trigrams(sequence, **kwargs):
     for item in ngrams(sequence, 3, **kwargs):
         yield item
 
-
 def everygrams(sequence, min_len=1, max_len=-1, **kwargs):
     """
     Returns all possible ngrams generated from a sequence of items, as an iterator.
@@ -1148,6 +1448,18 @@ def uninstall_package(package_name: str):
     import pip
     pip.main(['uninstall', package_name, '-y'])
 
+# A list of conda configuration commands.
+conda_mirror_commands=[
+    "pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple", # Windows recommended
+    "conda config --add channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/", # MacOS recommended
+    "conda config --add channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/free/",
+    "conda config --add channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/main/",
+    "conda config --append channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/fastai/",
+    "conda config --append channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/pytorch/",
+    "conda config --append channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/bioconda/",
+    "pip config set global.index-url https://mirrors.aliyun.com/pypi/simple/"
+]
+
 def DirList(root_dir: str) -> tuple: 
     """
     List the contents of a directory and return two lists containing the names of the directories and files in the directory.
@@ -1251,10 +1563,8 @@ def sort_strings_with_embedded_numbers(strings: list) -> list:
     """
     # Sort the strings using the extract_numbers() function as the key
     sorted_strings=sorted(strings, key=extract_numbers)
-
     return sorted_strings
 
-
 def run_command(command: str) -> str:
     """
     Run a command and return its output as a string.
@@ -1444,7 +1754,6 @@ class PGScraper(object):
             return all_want_list   
 
 
-
     
 # -*- coding: utf-8 -*-
 """
@@ -1490,7 +1799,6 @@ class ProgressData(object):
                   self.size, self.unit, progress, speed, self.unit))
             print('%50s'%('/'*int((1-progress)*50)))
 
-
 def levenshtein_distance(s, t):
 	m, n=len(s), len(t)
 	if m < n:
@@ -1516,7 +1824,7 @@ pgs_abbres_words=['A.B.','A.D.','A.G.','A.I.','A.M.','A.P.','A.V.','AFP.','Ala.'
 def clean_text(text): #清洗除了句号以外的其他标点符号问题
     # 在标点符号右边邻接单词前添加空格
     import re
-    text=replace_chinese_punctuation_with_english(text) 
+    # text=replace_chinese_punctuation_with_english(text) 
     text=re.sub(r'(?<=[\?\!\,\;\:\)\]\}])\s*(?=\w)', ' ', text)
     # 删除标点符号与左边单词之间的空格
     text=re.sub(r'\s*([\?\!\,\;\:\)\]\}\>])', r'\1', text)
@@ -1532,24 +1840,45 @@ def clean_text(text): #清洗除了句号以外的其他标点符号问题
 
 def clean_text_with_abbreviations(text):
     import re
-    text=clean_text(text)
-    matches=[]
-    for seg in text.split():
+
+    # 按行分割文本
+    lines = text.splitlines()
+
+    # 清洗每一行
+    cleaned_lines = []
+    for line in lines:
+        cleaned_line = clean_line_with_abbreviations(line)
+        cleaned_lines.append(cleaned_line)
+
+    # 将清洗后的行重新组合成文本
+    cleaned_text = '\n'.join(cleaned_lines)
+    return cleaned_text
+
+def clean_line_with_abbreviations(line):
+    import re
+
+    # 清洗除了句号以外的其他标点符号问题
+    line = clean_text(line)
+
+    matches = []
+    for seg in line.split():
         if "." in seg:
-            if seg.endswith(".") is False:
+            if not seg.endswith("."):
                 matches.append(seg)
             elif seg.endswith("..") and "..." not in seg:
-                text=text.replace("..", ".")
-    
+                line = line.replace("..", ".")
+
     for match in matches:
         if any(word in match for word in pgs_abbres_words):
-            inter=match.split(".")
-            new_match="".join([w+"." for w in inter[0:-1]])+" "+inter[-1]
-            text=text.replace(match, new_match)
+            inter = match.split(".")
+            new_match = "".join([w + "." for w in inter[0:-1]]) + " " + inter[-1]
+            line = line.replace(match, new_match)
         else:
-            text=text.replace(match, match.replace(".",". "))
-    text=re.sub(r'\s+\.', '.', text)
-    return text
+            line = line.replace(match, match.replace(".", ". "))
+
+    line = re.sub(r'\s+\.', '.', line)
+    return line
+
 
 import shutil
 def move_file(source_file, destination_folder, new_file_name=None):
@@ -1597,7 +1926,6 @@ def copy_file(source_file, destination_folder, new_file_name=None):
 
     # Copy the file to the destination folder
     shutil.copy2(source_file, destination_file)
-
     
 def check_empty_cells(file_path):
     """
@@ -1636,7 +1964,6 @@ def makefile(file_path):
     else:
         write_to_txt(file_path, "")
         
-        
 def save_dict_to_excel(data, output_file, headers=None):
     """
     Save Python dictionary data into an Excel .xlsx file with custom headers.
@@ -1845,4 +2172,57 @@ def get_stopwords(language=None):
         return en_stopwords
     else:
         lang_stopwords=get_data_lines(find_txt_files_with_keyword(stopwords_path, language)[0])
-        return lang_stopwords    
+        return lang_stopwords    
+
+from PIL import Image
+def replace_white_with_transparency(input_path, output_path):
+    """
+    This function opens an image, replaces all white pixels with transparent pixels.
+
+    Parameters:
+    input_path (str): The path to the input image file.
+    output_path (str): The path to save the output image file.
+    """
+    # 从RGB（24位）模式转成RGBA（32位）模式
+    img = Image.open(input_path).convert('RGBA')
+    W, L = img.size
+    white_pixel = (0, 0, 0, 0)  # white
+    for h in range(W):
+      for i in range(L):
+        if img.getpixel((h, i)) == white_pixel:
+          img.putpixel((h, i), (255, 255, 255, 0))   # make it transparent
+    img.save(output_path)
+
+def get_font_path(font_name=None):
+    '''
+    Retrieves the file path of a specified font.
+
+    Parameters
+    ----------
+    font_name : str, optional
+        The name of the font file (must end with ".ttf"). If provided, it should match one of the available fonts in the library, such as:
+        - 'DejaVuSans.ttf'
+        - '书体坊赵九江钢笔行书体.ttf'
+        - '全新硬笔楷书简.ttf'
+        - '全新硬笔行书简.ttf'
+        - '博洋行书3500.TTF'
+        - '陆柬之行书字体.ttf'
+        The default is None, which will return the path for 'DejaVuSans.ttf'.
+
+    Returns
+    -------
+    font_path : str
+        The full file path of the specified font. If no font name is provided, the default path for 'DejaVuSans.ttf' will be returned.
+        Example: "C:/Windows/Fonts/simhei.ttf"
+    '''
+    
+    font_folder = get_library_location("PgsFile") + "/PgsFile/models/fonts"
+    if font_name is None:
+        font_path = get_full_path(font_folder, "DejaVuSans.ttf")
+    else:
+        font_path = get_full_path(font_folder, font_name)
+    return font_path
+
+simhei_default_font_path_MacOS_Windows=["/System/Library/Fonts/STHeiti Medium.ttc",
+                   r"C:\Windows\Fonts\simhei.ttf",  # Use a font that supports Chinese characters
+                   ]

PgsFile/__init__.py

@@ -7,6 +7,7 @@ from .PgsFile import headers, encode_chinese_keyword_for_url
 from .PgsFile import install_package, uninstall_package
 from .PgsFile import run_script, run_command
 from .PgsFile import get_library_location
+from .PgsFile import conda_mirror_commands
 
 # 3. Text data retrieval
 from .PgsFile import get_data_text, get_data_lines, get_json_lines, get_tsv_lines
@@ -19,7 +20,7 @@ from .PgsFile import write_to_txt, write_to_excel, write_to_json, write_to_json_
 
 # 5. File/folder process
 from .PgsFile import FilePath, FileName, DirList
-from .PgsFile import get_subfolder_path
+from .PgsFile import get_subfolder_path, get_full_path
 from .PgsFile import makedirec, makefile
 from .PgsFile import source_path, next_folder_names, get_directory_tree_with_meta, find_txt_files_with_keyword
 from .PgsFile import remove_empty_folders, remove_empty_txts, remove_empty_lines, remove_empty_last_line, move_file, copy_file
@@ -32,18 +33,23 @@ from .PgsFile import nltk_en_tags, nltk_tag_mapping, thulac_tags, ICTCLAS2008, L
 from .PgsFile import check_contain_chinese, check_contain_number
 from .PgsFile import replace_chinese_punctuation_with_english
 from .PgsFile import replace_english_punctuation_with_chinese
-from .PgsFile import clean_list, clean_text_with_abbreviations
+from .PgsFile import clean_list, clean_text, clean_text_with_abbreviations, clean_line_with_abbreviations
 from .PgsFile import extract_chinese_punctuation, generate_password, sort_strings_with_embedded_numbers
 
 # 7. NLP (natural language processing)
 from .PgsFile import strQ2B_raw, strQ2B_words
 from .PgsFile import ngrams, bigrams, trigrams, everygrams, compute_similarity
 from .PgsFile import word_list, batch_word_list
-from .PgsFile import cs, cs1, sent_tokenize, word_tokenize
+from .PgsFile import cs, cs1, sent_tokenize, word_tokenize, word_tokenize2
 
 # 8. Maths
 from .PgsFile import len_rows, check_empty_cells
 from .PgsFile import format_float, decimal_to_percent, Percentage
 from .PgsFile import get_text_length_kb, extract_numbers
 
+# 9. Visualization
+from .PgsFile import replace_white_with_transparency
+from .PgsFile import simhei_default_font_path_MacOS_Windows
+from .PgsFile import get_font_path
+
 name = "PgsFile"

PgsFile/models/model_reviews_ReadMe.txt

@@ -0,0 +1,134 @@
+model_1.0.bin ['samples: 30', 'precision: 0.7666666666666667', 'recall: 0.696969696969697', 'F1: 0.7301587301587302'] 
+model_1.2.bin ['samples: 30', 'precision: 0.8333333333333334', 'recall: 0.7575757575757576', 'F1: 0.7936507936507938'] 
+model_1.4.bin ['samples: 30', 'precision: 0.8333333333333334', 'recall: 0.7575757575757576', 'F1: 0.7936507936507938'] 
+model_1.5.bin ['samples: 30', 'precision: 0.8333333333333334', 'recall: 0.7575757575757576', 'F1: 0.7936507936507938'] 
+model_1.6.bin ['samples: 30', 'precision: 0.9', 'recall: 0.8181818181818182', 'F1: 0.8571428571428572'] 
+model_1.7.bin ['samples: 30', 'precision: 0.8666666666666667', 'recall: 0.7878787878787878', 'F1: 0.8253968253968254'] 
+model_1.8.bin ['samples: 30', 'precision: 0.8', 'recall: 0.7272727272727273', 'F1: 0.761904761904762']
+model_1.9.bin ['samples: 30', 'precision: 0.8', 'recall: 0.7272727272727273', 'F1: 0.761904761904762'] 
+model_2.0.bin ['samples: 30', 'precision: 0.8333333333333334', 'recall: 0.7575757575757576', 'F1: 0.7936507936507938'] 
+model_2.1.bin ['samples: 30', 'precision: 0.8666666666666667', 'recall: 0.7878787878787878', 'F1: 0.8253968253968254'] 
+
+
+model_1.0.bin ['samples: 292', 'precision: 0.5787671232876712', 'recall: 0.48011363636363635', 'F1: 0.5248447204968945'] 
+model_1.2.bin ['samples: 292', 'precision: 0.636986301369863', 'recall: 0.5284090909090909', 'F1: 0.577639751552795'] 
+model_1.4.bin ['samples: 292', 'precision: 0.7191780821917808', 'recall: 0.5965909090909091', 'F1: 0.6521739130434782'] 
+model_1.5.bin ['samples: 292', 'precision: 0.6815068493150684', 'recall: 0.5653409090909091', 'F1: 0.6180124223602484'] 
+model_1.6.bin ['samples: 292', 'precision: 0.726027397260274', 'recall: 0.6022727272727273', 'F1: 0.6583850931677019'] 
+model_1.7.bin ['samples: 292', 'precision: 0.7363013698630136', 'recall: 0.6107954545454546', 'F1: 0.6677018633540373'] 
+model_1.8.bin ['samples: 292', 'precision: 0.7431506849315068', 'recall: 0.6164772727272727', 'F1: 0.6739130434782609'] 
+model_1.9.bin ['samples: 292', 'precision: 0.7773972602739726', 'recall: 0.6448863636363636', 'F1: 0.7049689440993789'] 
+model_2.0.bin ['samples: 292', 'precision: 0.7636986301369864', 'recall: 0.6335227272727273', 'F1: 0.6925465838509317'] 
+model_2.1.bin ['samples: 292', 'precision: 0.7671232876712328', 'recall: 0.6363636363636364', 'F1: 0.6956521739130435'] 
+
+
+model_1.0.bin ['samples: 322', 'precision: 0.5962732919254659', 'recall: 0.4987012987012987', 'F1: 0.5431400282885432']
+model_1.2.bin ['samples: 322', 'precision: 0.65527950310559', 'recall: 0.548051948051948', 'F1: 0.5968882602545968'] 
+model_1.4.bin ['samples: 322', 'precision: 0.7267080745341615', 'recall: 0.6077922077922078', 'F1: 0.6619519094766619']
+model_1.5.bin ['samples: 322', 'precision: 0.6956521739130435', 'recall: 0.5818181818181818', 'F1: 0.6336633663366337'] 
+model_1.6.bin ['samples: 322', 'precision: 0.7422360248447205', 'recall: 0.6207792207792208', 'F1: 0.6760961810466761'] 
+model_1.7.bin ['samples: 322', 'precision: 0.7484472049689441', 'recall: 0.625974025974026', 'F1: 0.6817538896746819'] 
+model_1.8.bin ['samples: 322', 'precision: 0.7484472049689441', 'recall: 0.625974025974026', 'F1: 0.6817538896746819'] 
+model_1.9.bin ['samples: 322', 'precision: 0.7795031055900621', 'recall: 0.6519480519480519', 'F1: 0.71004243281471'] 
+model_2.0.bin ['samples: 322', 'precision: 0.7701863354037267', 'recall: 0.6441558441558441', 'F1: 0.7015558698727016']
+model_2.1.bin ['samples: 322', 'precision: 0.7763975155279503', 'recall: 0.6493506493506493', 'F1: 0.7072135785007072'] 
+
+
+=========================================================非重复验证集==================================================
+
+model_1.2.bin ['samples: 303', 'precision: 0.6435643564356436', 'recall: 0.5342465753424658', 'F1: 0.5838323353293414']
+model_1.4.bin ['samples: 303', 'precision: 0.7161716171617162', 'recall: 0.5945205479452055', 'F1: 0.6497005988023953']
+model_1.5.bin ['samples: 303', 'precision: 0.6864686468646864', 'recall: 0.5698630136986301', 'F1: 0.6227544910179641'] 
+model_1.6.bin ['samples: 303', 'precision: 0.7326732673267327', 'recall: 0.6082191780821918', 'F1: 0.6646706586826348'] 
+model_1.7.bin ['samples: 303', 'precision: 0.7425742574257426', 'recall: 0.6164383561643836', 'F1: 0.6736526946107784'] 
+model_1.8.bin ['samples: 303', 'precision: 0.7392739273927392', 'recall: 0.6136986301369863', 'F1: 0.6706586826347306'] 
+model_1.9.bin ['samples: 303', 'precision: 0.7722772277227723', 'recall: 0.6410958904109589', 'F1: 0.7005988023952096']
+model_2.0.bin ['samples: 303', 'precision: 0.759075907590759', 'recall: 0.6301369863013698', 'F1: 0.688622754491018'] 
+model_2.1.bin ['samples: 303', 'precision: 0.7623762376237624', 'recall: 0.6328767123287671', 'F1: 0.6916167664670658'] 
+model_2.2.bin ['samples: 303', 'precision: 0.7458745874587459', 'recall: 0.6191780821917808', 'F1: 0.6766467065868264'] 
+
+=================================================非重复验证集+5分标签==================================================
+
+model_1.2.bin ['samples: 30', 'precision: 0.8333333333333334', 'recall: 0.78125', 'F1: 0.8064516129032259'] 
+model_1.4.bin ['samples: 30', 'precision: 0.8666666666666667', 'recall: 0.8125', 'F1: 0.8387096774193549'] 
+model_1.5.bin ['samples: 30', 'precision: 0.9', 'recall: 0.84375', 'F1: 0.870967741935484'] 
+model_1.6.bin ['samples: 30', 'precision: 0.9', 'recall: 0.84375', 'F1: 0.870967741935484'] 
+model_1.7.bin ['samples: 30', 'precision: 0.8', 'recall: 0.75', 'F1: 0.7741935483870969'] 
+model_1.8.bin ['samples: 30', 'precision: 0.8333333333333334', 'recall: 0.78125', 'F1: 0.8064516129032259'] 
+model_1.9.bin ['samples: 30', 'precision: 0.8333333333333334', 'recall: 0.78125', 'F1: 0.8064516129032259'] 
+model_2.0.bin ['samples: 30', 'precision: 0.8333333333333334', 'recall: 0.78125', 'F1: 0.8064516129032259'] 
+model_2.1.bin ['samples: 30', 'precision: 0.9', 'recall: 0.84375', 'F1: 0.870967741935484'] 
+model_2.2.bin ['samples: 30', 'precision: 0.9', 'recall: 0.84375', 'F1: 0.870967741935484'] 
+
+
+model_1.2.bin ['samples: 302', 'precision: 0.6721854304635762', 'recall: 0.6444444444444445', 'F1: 0.6580226904376014'] 
+model_1.4.bin ['samples: 302', 'precision: 0.7019867549668874', 'recall: 0.6730158730158731', 'F1: 0.6871961102106969'] 
+model_1.5.bin ['samples: 302', 'precision: 0.7185430463576159', 'recall: 0.6888888888888889', 'F1: 0.7034035656401946'] 
+model_1.6.bin ['samples: 302', 'precision: 0.7086092715231788', 'recall: 0.6793650793650794', 'F1: 0.6936790923824959'] 
+model_1.7.bin ['samples: 302', 'precision: 0.7052980132450332', 'recall: 0.6761904761904762', 'F1: 0.6904376012965965'] 
+model_1.8.bin ['samples: 302', 'precision: 0.7317880794701986', 'recall: 0.7015873015873015', 'F1: 0.7163695299837927'] 
+model_1.9.bin ['samples: 302', 'precision: 0.7317880794701986', 'recall: 0.7015873015873015', 'F1: 0.7163695299837927'] 
+model_2.0.bin ['samples: 302', 'precision: 0.7417218543046358', 'recall: 0.7111111111111111', 'F1: 0.7260940032414911'] 
+model_2.1.bin ['samples: 302', 'precision: 0.7516556291390728', 'recall: 0.7206349206349206', 'F1: 0.7358184764991895'] 
+model_2.2.bin ['samples: 302', 'precision: 0.7582781456953642', 'recall: 0.726984126984127', 'F1: 0.7423014586709886'] 
+
+
+model_1.2.bin ['samples: 303', 'precision: 0.6732673267326733', 'recall: 0.6455696202531646', 'F1: 0.6591276252019386'] 
+model_1.4.bin ['samples: 303', 'precision: 0.7029702970297029', 'recall: 0.6740506329113924', 'F1: 0.6882067851373183'] 
+model_1.5.bin ['samples: 303', 'precision: 0.7194719471947195', 'recall: 0.689873417721519', 'F1: 0.7043618739903069'] 
+model_1.6.bin ['samples: 303', 'precision: 0.7095709570957096', 'recall: 0.680379746835443', 'F1: 0.6946688206785137'] 
+model_1.7.bin ['samples: 303', 'precision: 0.7062706270627063', 'recall: 0.6772151898734177', 'F1: 0.6914378029079159'] 
+model_1.8.bin ['samples: 303', 'precision: 0.7326732673267327', 'recall: 0.7025316455696202', 'F1: 0.7172859450726979'] 
+model_1.9.bin ['samples: 303', 'precision: 0.7326732673267327', 'recall: 0.7025316455696202', 'F1: 0.7172859450726979'] 
+model_2.0.bin ['samples: 303', 'precision: 0.7425742574257426', 'recall: 0.7120253164556962', 'F1: 0.7269789983844911'] 
+model_2.1.bin ['samples: 303', 'precision: 0.7524752475247525', 'recall: 0.7215189873417721', 'F1: 0.7366720516962842'] 
+model_2.2.bin ['samples: 303', 'precision: 0.759075907590759', 'recall: 0.7278481012658228', 'F1: 0.7431340872374799'] 
+
+
+model_1.2.bin ['samples: 425', 'precision: 0.6470588235294118', 'recall: 0.5456349206349206', 'F1: 0.5920344456404736'] 
+model_1.2.bin ['samples: 425', 'precision: 0.691764705882353', 'recall: 0.6621621621621622', 'F1: 0.6766398158803222'] 
+model_1.4.bin ['samples: 425', 'precision: 0.7129411764705882', 'recall: 0.6824324324324325', 'F1: 0.6973532796317606'] 
+model_1.5.bin ['samples: 425', 'precision: 0.7294117647058823', 'recall: 0.6981981981981982', 'F1: 0.713463751438435'] 
+model_1.6.bin ['samples: 425', 'precision: 0.7129411764705882', 'recall: 0.6824324324324325', 'F1: 0.6973532796317606'] 
+model_1.7.bin ['samples: 425', 'precision: 0.7105882352941176', 'recall: 0.6801801801801802', 'F1: 0.6950517836593786'] 
+model_1.8.bin ['samples: 425', 'precision: 0.7505882352941177', 'recall: 0.7184684684684685', 'F1: 0.7341772151898734'] 
+model_1.9.bin ['samples: 425', 'precision: 0.7529411764705882', 'recall: 0.7207207207207207', 'F1: 0.7364787111622554'] 
+model_2.0.bin ['samples: 425', 'precision: 0.7670588235294118', 'recall: 0.7342342342342343', 'F1: 0.7502876869965478'] 
+model_2.1.bin ['samples: 425', 'precision: 0.7717647058823529', 'recall: 0.7387387387387387', 'F1: 0.7548906789413118'] 
+model_2.2.bin ['samples: 425', 'precision: 0.7764705882352941', 'recall: 0.7432432432432432', 'F1: 0.7594936708860759'] 
+
+model_1.2.bin ['samples: 447', 'precision: 0.6935123042505593', 'recall: 0.6623931623931624', 'F1: 0.6775956284153005'] 
+model_1.4.bin ['samples: 447', 'precision: 0.7158836689038032', 'recall: 0.6837606837606838', 'F1: 0.6994535519125684'] 
+model_1.5.bin ['samples: 447', 'precision: 0.7337807606263982', 'recall: 0.7008547008547008', 'F1: 0.7169398907103826'] 
+model_1.6.bin ['samples: 447', 'precision: 0.7203579418344519', 'recall: 0.688034188034188', 'F1: 0.7038251366120218'] 
+model_1.7.bin ['samples: 447', 'precision: 0.7158836689038032', 'recall: 0.6837606837606838', 'F1: 0.6994535519125684'] 
+model_1.8.bin ['samples: 447', 'precision: 0.7539149888143176', 'recall: 0.7200854700854701', 'F1: 0.7366120218579234'] 
+model_1.9.bin ['samples: 447', 'precision: 0.7539149888143176', 'recall: 0.7200854700854701', 'F1: 0.7366120218579234'] 
+model_2.0.bin ['samples: 447', 'precision: 0.7695749440715883', 'recall: 0.7350427350427351', 'F1: 0.7519125683060108'] 
+model_2.1.bin ['samples: 447', 'precision: 0.7718120805369127', 'recall: 0.7371794871794872', 'F1: 0.7540983606557377'] 
+model_2.2.bin ['samples: 447', 'precision: 0.7785234899328859', 'recall: 0.7435897435897436', 'F1: 0.760655737704918'] 
+
+model_1.2.bin  
+model_1.4.bin 
+model_1.5.bin 
+model_1.6.bin 
+model_1.7.bin 
+model_1.8.bin 
+model_1.9.bin 
+model_2.0.bin 
+model_2.1.bin 
+model_2.2.bin
+
+model_1.2.bin 
+model_1.4.bin 
+model_1.5.bin 
+model_1.6.bin 
+model_1.7.bin 
+model_1.8.bin 
+model_1.9.bin 
+model_2.0.bin 
+model_2.1.bin 
+model_2.2.bin
+
+
+

PgsFile-0.2.4.dist-info/METADATA

@@ -0,0 +1,41 @@
+Metadata-Version: 2.1
+Name: PgsFile
+Version: 0.2.4
+Summary: This module streamlines Python package management, script execution, file handling, web scraping, multimedia downloads, data cleaning, and NLP tasks such as word tokenization and POS tagging. It also assists with generating word lists and plotting data, making these tasks more accessible and convenient for literary students. Whether you need to scrape data from websites, clean text, or analyze language, this module provides user-friendly tools to simplify your workflow.
+Home-page: https://mp.weixin.qq.com/s/12-KVLfaPszoZkCxuRd-nQ?token=1589547443&lang=zh_CN
+Author: Pan Guisheng
+Author-email: 895284504@qq.com
+License: Educational free
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: Free For Educational Use
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: chardet
+Requires-Dist: pandas
+Requires-Dist: python-docx
+Requires-Dist: pip
+Requires-Dist: requests
+Requires-Dist: fake-useragent
+Requires-Dist: lxml
+Requires-Dist: pimht
+Requires-Dist: pysbd
+Requires-Dist: nlpir-python
+Requires-Dist: pillow
+
+Purpose: This module is designed to make complex tasks accessible and convenient, even for beginners. By providing a unified set of tools, it simplifies the workflow for data collection, processing, and analysis. Whether you're scraping data from the web, cleaning text, or performing NLP tasks, this module ensures you can focus on your research without getting bogged down by technical challenges.
+
+Key Features:
+1. Web Scraping: Easily scrape data from websites and download multimedia content.
+2. Package Management: Install, uninstall, and manage Python packages with simple commands.
+3. Data Retrieval: Extract data from various file formats like text, JSON, TSV, Excel, and HTML (both online and offline).
+4. Data Storage: Write and append data to text files, Excel, JSON, and JSON lines.
+5. File and Folder Processing: Manage file paths, create directories, move or copy files, and search for files with specific keywords.
+6. Data Cleaning: Clean text, handle punctuation, remove stopwords, and prepare data for analysis.
+7. NLP: Perform tokenization, generate n-grams, and create word lists for text analysis.
+8. Math Operations: Format numbers, convert decimals to percentages, and validate data.
+9. Visualization: Process images (e.g., make white pixels transparent) and manage fonts for rendering text.
+
+Author: Pan Guisheng, a PhD student at the Graduate Institute of Interpretation and Translation of Shanghai International Studies University
+E-mail: 895284504@qq.com

{PgsFile-0.2.3.dist-info → PgsFile-0.2.4.dist-info}/RECORD

@@ -1,5 +1,5 @@
-PgsFile/PgsFile.py,sha256=MpXQK6MLMBh1JMAcBw5sRiRof--x4OyARcCsWwn7Z4A,85828
-PgsFile/__init__.py,sha256=E4VfPu1BxCBcZ5WXi5E6faPaNt_Shpvgh9LvBlg7eA0,2389
+PgsFile/PgsFile.py,sha256=V_Pnn5hljeR9xYQ8hyUAmf92140N4ORoQOe-cdBHJos,101212
+PgsFile/__init__.py,sha256=C-uX4tN3J3L5Zr6r8qQx0zwNaG0UXTowxK_K0pd0Jt4,2680
 PgsFile/Corpora/Idioms/English_Idioms_8774.txt,sha256=qlsP0yI_XGECBRiPZuLkGZpdasc77sWSKexANu7v8_M,175905
 PgsFile/Corpora/Monolingual/Chinese/People's Daily 20130605/Raw/00000000.txt,sha256=SLGGSMSb7Ff1RoBstsTW3yX2wNZpqEUchFNpcI-mrR4,1513
 PgsFile/Corpora/Monolingual/Chinese/People's Daily 20130605/Raw/00000001.txt,sha256=imOa6UoCOIZoPXT4_HNHgCUJtd4FTIdk2FZNHNBgJyg,3372
@@ -2600,6 +2600,7 @@ PgsFile/Corpora/Stopwords/turkish.txt,sha256=uGUvjEm2GR8PuVY_JeHNxhD7cWlNlF7vc3V
 PgsFile/Corpora/Stopwords/ukrainian.txt,sha256=fEzWLTwnWJriILkO-5jSfE2SpqY-GPf_kR4zid3MFUI,4131
 PgsFile/Corpora/Stopwords/vietnamese.txt,sha256=88yRtVMaRSFqas1iGGa6kOGDCZTgtzRPmR3q9dHshdc,20485
 PgsFile/Corpora/Terminology/Chinese_Thought.json,sha256=CdkuF2wLaDC5V3sRefcU1RZwXm4-wTZ-Qfk8r7gsu8I,2301866
+PgsFile/models/NLPIR.user,sha256=DykLJdr8_cVHrdCnDJES1O5dgmnYqfaSO1_dtAVKYJk,3356
 PgsFile/models/czech.pickle,sha256=W6c9KTx9eVOVa88C82lexcHw1Sfyo8OAl_VZM5T6FpA,1265552
 PgsFile/models/danish.pickle,sha256=6il2CgqRl_UspZ54rq_FpvVdBSWPr32xcJsrnrMh7yA,1264725
 PgsFile/models/dutch.pickle,sha256=So4ms9aMRcOOWU0Z4tVndEe_3KpjbTsees_tDpJy1zw,742624
@@ -2611,6 +2612,8 @@ PgsFile/models/german.pickle,sha256=6rSX-ghUExMMj9D7E7kpEokwr-L2om6ocVyV33CI6Xw,
 PgsFile/models/greek.pickle,sha256=IXUqZ2L61c_kb7XEX62ahUhKDo6Bxn5q9vuXPPwn1nw,1953106
 PgsFile/models/italian.pickle,sha256=3LJxfXvl8m6GCpLgWs9psRI6X0UnzXommpq56eZoyAU,658331
 PgsFile/models/malayalam.pickle,sha256=H4z1isvbf0cqxAr_wTZjvkLa-0fBUDDBGt4ERMng5T0,221207
+PgsFile/models/model_reviews2.2.bin,sha256=D6uL8KZIxD0rfWjH0kYEb7z_HE4aTJXpj82HzsCOpuk,1943196
+PgsFile/models/model_reviews_ReadMe.txt,sha256=Q9uLJwudMmsTKfd11l1tOcIP8lwsemIwnAVJG_3SYjU,11433
 PgsFile/models/norwegian.pickle,sha256=5Kl_j5oDoDON10a8yJoK4PVK5DuDX6N9g-J54cp5T68,1259779
 PgsFile/models/polish.pickle,sha256=FhJ7bRCTNCej6Q-yDpvlPh-zcf95pzDBAwc07YC5DJI,2042451
 PgsFile/models/portuguese.pickle,sha256=uwG_fHmk6twheLvSCWZROaDks48tHET-8Jfek5VRQOA,649051
@@ -2619,8 +2622,14 @@ PgsFile/models/slovene.pickle,sha256=faxlAhKzeHs5mWwBvSCEEVST5vbsOQurYfdnUlsIuOo
 PgsFile/models/spanish.pickle,sha256=Jx3GAnxKrgVvcqm_q1ZFz2fhmL9PlyiVhE5A9ZiczcM,597831
 PgsFile/models/swedish.pickle,sha256=QNUOva1sqodxXy4wCxIX7JLELeIFpUPMSlaQO9LJrPo,1034496
 PgsFile/models/turkish.pickle,sha256=065H12UB0CdpiAnRLnUpLJw5KRBIhUM0KAL5Xbl2XMw,1225013
-PgsFile-0.2.3.dist-info/LICENSE,sha256=cE5c-QToSkG1KTUsU8drQXz1vG0EbJWuU4ybHTRb5SE,1138
-PgsFile-0.2.3.dist-info/METADATA,sha256=a9KMN6LpC2raZYhWwrFhWCXKl7nWneiXT7KtvA74ruY,5070
-PgsFile-0.2.3.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-PgsFile-0.2.3.dist-info/top_level.txt,sha256=028hCfwhF3UpfD6X0rwtWpXI1RKSTeZ1ALwagWaSmX8,8
-PgsFile-0.2.3.dist-info/RECORD,,
+PgsFile/models/fonts/DejaVuSans.ttf,sha256=faGVp0xVvvmI0NSPlQi9XYSUJcF3Dbpde_xs6e2EiVQ,757076
+PgsFile/models/fonts/书体坊赵九江钢笔行书体.ttf,sha256=fTOv4FFMnYtN1zCZghJ6-P1pzznA5qqoujwpDFY63Ek,3140656
+PgsFile/models/fonts/全新硬笔楷书简.ttf,sha256=mPemGYMpgQxvFL1pFjjnyUMIprHzcoOaw8oeZQ4k1x0,2397296
+PgsFile/models/fonts/全新硬笔行书简.ttf,sha256=bUtbl71eK_ellp1z0tCmmR_P-JhqVFIpzeuRlrEBo9g,2611516
+PgsFile/models/fonts/博洋行书3500.TTF,sha256=VrgeHr8cgOL6JD05QyuD9ZSyw4J2aIVxKxW8zSajq6Q,4410732
+PgsFile/models/fonts/陆柬之行书字体.ttf,sha256=Zpd4Z7E9w-Qy74yklXHk4vM7HOtHuQgllvygxZZ1Hvs,1247288
+PgsFile-0.2.4.dist-info/LICENSE,sha256=cE5c-QToSkG1KTUsU8drQXz1vG0EbJWuU4ybHTRb5SE,1138
+PgsFile-0.2.4.dist-info/METADATA,sha256=JC1a8Xrh3tDt5-HNnCJY3V4tNYKstu83V2qo_FqkATY,2711
+PgsFile-0.2.4.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+PgsFile-0.2.4.dist-info/top_level.txt,sha256=028hCfwhF3UpfD6X0rwtWpXI1RKSTeZ1ALwagWaSmX8,8
+PgsFile-0.2.4.dist-info/RECORD,,

PgsFile-0.2.3.dist-info/METADATA

@@ -1,79 +0,0 @@
-Metadata-Version: 2.1
-Name: PgsFile
-Version: 0.2.3
-Summary: This module aims to simplify Python package management, script execution, file handling, web scraping, multimedia download, data cleaning, NLP tasks like Chinese word tokenization and POS tagging, and word list generation for literary students, making it more accessible and convenient to use.
-Home-page: https://mp.weixin.qq.com/s/12-KVLfaPszoZkCxuRd-nQ?token=1589547443&lang=zh_CN
-Author: Pan Guisheng
-Author-email: 895284504@qq.com
-License: Educational free
-Classifier: Programming Language :: Python :: 3
-Classifier: License :: Free For Educational Use
-Classifier: Operating System :: OS Independent
-Requires-Python: >=3.8
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: chardet
-Requires-Dist: pandas
-Requires-Dist: python-docx
-Requires-Dist: pip
-Requires-Dist: requests
-Requires-Dist: fake-useragent
-Requires-Dist: lxml
-Requires-Dist: pimht
-Requires-Dist: pysbd
-Requires-Dist: nlpir-python
-
-Purpose: This module aims to assist Python beginners, particularly instructors and students of foreign languages and literature, by providing a convenient way to manage Python packages, run Python scripts, and perform operations on various file types such as txt, xlsx, json, tsv, html, mhtml, and docx. It also includes functionality for data scraping, cleaning and generating word lists.
-
-
-Function 1: Enables efficient data retrieval and storage in files with a single line of code.
-
-Function 2: Facilitates retrieval of all absolute file paths and file names in any folder (including sub-folders) with a single line of code using "FilePath" and "FileName" functions.
-
-Function 3: Simplifies creation of word lists and frequency sorting from a file or batch of files using "word_list" and "batch_word_list" functions in PgsFile.
-
-Function 4: Pgs-Corpora is a comprehensive language resource included in this library, featuring a monolingual corpus of native and translational Chinese and native and non-native English, as well as a bi-directional parallel corpus of Chinese and English texts covering financial, legal, political, academic, and sports news topics. Additionally, the library includes a collection of 8774 English idioms, stopwords for 28 languages, and a termbank of Chinese thought and culture.
-
-Function 5: This library provides support for common text cleaning tasks, such as removing empty text, empty lines, and folders containing empty text. It also offers functions for converting full-width characters to half-width characters and vice versa, as well as standardizing the format of Chinese and English punctuation. These features can help improve the quality and consistency of text data used in natural language processing tasks.
-
-Function 6: It also manages Python package installations and uninstallations, and allows running scripts and commands in Python interactive command lines instead of Windows command prompt.
-
-Function 7: Download audiovisual files like videos, images, and audio using audiovisual_downloader, which is extremely useful and efficient. Additionally, scrape newspaper data with PGScraper, a highly efficient tool for this purpose.
-
-Table 1: The directory and size of Pgs-Corpora
-├── Idioms (1, 171.78 KB)
-├── Monolingual (2197, 63.65 MB)
-│   ├── Chinese (456, 15.27 MB)
-│   │   ├── People's Daily 20130605 (396, 1.38 MB)
-│   │   │   ├── Raw (132, 261.73 KB)
-│   │   │   ├── Seg_only (132, 471.47 KB)
-│   │   │   └── Tagged (132, 675.30 KB)
-│   │   └── Translational Fictions (60, 13.89 MB)
-│   └── English (1741, 48.38 MB)
-│       ├── Native (65, 44.14 MB)
-│       │   ├── A Short Collection of British Fiction (27, 33.90 MB)
-│       │   └── Preschoolers- and Teenagers-oriented Texts in English (36, 10.24 MB)
-│       ├── Non-native (1675, 3.63 MB)
-│       │   └── Shanghai Daily (1675, 3.63 MB)
-│       │       └── Business_2019 (1675, 3.63 MB)
-│       │           ├── 2019-01-01 (1, 3.35 KB)
-│       │           ├── 2019-01-02 (1, 3.65 KB)
-│       │           ├── 2019-01-03 (7, 10.90 KB)
-│       │           ├── 2019-01-04 (5, 9.63 KB)
-│       │           └── 2019-01-07 (4, 9.50 KB)
-│       │           └── ... (and 245 more directories)
-│       └── Translational (1, 622.57 KB)
-├── Parallel (371, 24.67 MB)
-│   ├── HK Financial and Legal EC Parallel Corpora (5, 19.17 MB)
-│   ├── New Year Address_CE_2006-2021 (15, 147.49 KB)
-│   ├── Sports News_CE_2010 (20, 66.42 KB)
-│   ├── TED_EC_2017-2020 (330, 5.24 MB)
-│   └── Xi's Speech_CE_2021 (1, 53.01 KB)
-├── Stopwords (28, 88.09 KB)
-└── Terminology (1, 2.20 MB)
-
-...
-
-
-Author: Pan Guisheng, a PhD student at the Graduate Institute of Interpretation and Translation of Shanghai International Studies University
-E-mail: 895284504@qq.com