PgsFile 0.2.3__py3-none-any.whl → 0.2.5__py3-none-any.whl

PgsFile/PgsFile.py

@@ -103,7 +103,7 @@ def get_data_text(path):
     else:
         return None
 
-def get_data_lines(path):
+def get_data_lines(path, no_line_breaks=False):
     '''
     Parameters
     ----------
@@ -133,7 +133,10 @@ def get_data_lines(path):
     # Read the entire file using the detected encoding
     if encoding:
         with open(path, 'r', encoding=encoding, errors="ignore") as f:
-            lines = [l.strip() for l in f.readlines() if len(l.strip()) != 0]
+            if no_line_breaks is False:
+                lines = [l.strip() for l in f.readlines() if len(l.strip()) != 0]
+            else:
+                lines = f.readlines()
             return lines
     else:
         return None
@@ -197,13 +200,51 @@ def get_data_excel(excel_path,column_id,sheet_name=None):
     inter=df.iloc[0:,column_id] #提取第二列所有行  
     return list(inter)
 
-def write_to_excel(excel_path,dic_of_list,sheet_name=None,index=None):
+def write_to_excel(excel_path, data, sheet_name=None, index=None):
     '''
     Parameters
     ----------
     excel_path : TYPE
         DESCRIPTION. results.xlsx
         
+    data : TYPE, dict
+        DESCRIPTION. data = {'翻译': 24, '教学': 8, '数智': 6, '时代': 6, '财经': 6, '新': 4}
+        
+    sheet_name : TYPE, optional
+        DESCRIPTION. The default is None.
+        
+    index : TYPE, optional
+        DESCRIPTION. The default is None.
+        
+    Returns
+    -------
+    None.
+
+    '''
+    import pandas as pd
+    if sheet_name is None:
+        sheet_name="sheet1"
+    else:
+       sheet_name=sheet_name
+    if index is None:
+        index=False
+    else:
+        index=True        
+
+    col = list(data.keys())
+    freq = list(data.values())
+    dic_of_list={"items": col, "counts": freq}
+        
+    df=pd.DataFrame(dic_of_list)
+    df.style.to_excel(excel_path, sheet_name=sheet_name,startcol=0, index=index)
+
+def write_to_excel_normal(excel_path,dic_of_list,sheet_name=None,index=None):
+    '''
+    Parameters
+    ----------
+    excel_path : TYPE
+        DESCRIPTION. D:\results.xlsx
+        
     dic_of_list : TYPE
         DESCRIPTION. {"col":["a","b","c","d"],"freq":[1,2,3,4]}
         
@@ -471,6 +512,18 @@ def get_directory_tree_with_meta(start_path, indent='', show_meta=False, max_dir
         print(f"{indent}└── ... (and {remaining_directories} more directories)")
         # current_level=-1 will show all folders' info.
 
+def get_full_path(*path_components):
+    """
+    Combines multiple path components into a single, full path using os.path.join.
+
+    Args:
+        *path_components: Variable number of path components (strings).
+
+    Returns:
+        str: The combined full path.
+    """
+    return os.path.join(*path_components)
+
 def get_subfolder_path(parent_folder, subfolder_name):
     import os
     subfolder_name=subfolder_name.strip()
@@ -553,7 +606,6 @@ def batch_word_list(input_root):
     sorted_words=sorted(word_counts.items(), key=lambda x: x[1], reverse=True)
     return sorted_words 
 
-	
 def clean_list(meta):
     """
     Parameters
@@ -576,7 +628,6 @@ def clean_list(meta):
 
 yhd=["Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36','Mozilla/4.0 (compatible; MSIE 6.0; ) Opera/UCWEB7.0.2.37/28/999','Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)','Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)','Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)','Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser)','Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)','Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)','Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)','Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)','Mozilla/5.0 (BlackBerry; U; BlackBerry 9800; en) AppleWebKit/534.1+ (KHTML, like Gecko) Version/6.0.0.337 Mobile Safari/534.1+','Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0','Mozilla/5.0 (compatible; MSIE 9.0; Windows Phone OS 7.5; Trident/5.0; IEMobile/9.0; HTC; Titan)','Mozilla/5.0 (iPad; U; CPU OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5','Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5','Mozilla/5.0 (iPod; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5','Mozilla/5.0 (Linux; U; Android 2.3.7; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1','Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13','Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1','Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1','Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50','Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6','NOKIA5700/ UCWEB7.0.2.37/28/999','Openwave/ UCWEB7.0.2.37/28/999','Opera/9.80 (Android 2.3.4; Linux; Opera Mobi/build-1107180945; U; en-GB) Presto/2.8.149 Version/11.10','Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11','Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11','UCWEB7.0.2.37/28/999']
 
-
 def source_path(relative_path):
     import sys,os
     if getattr(sys, 'frozen', False): 
@@ -590,7 +641,6 @@ def next_folder_names(folder):
     folder_namelist=next(os.walk(folder))[1]
     return folder_namelist
 
-
 def remove_empty_txts(folder_path):
     import os
     files=FilePath(folder_path)
@@ -797,7 +847,6 @@ def cs1(text):
         sentences=sentences
     return sentences
 
-                
 def word_tokenize(text, pos_tagged=False):
     '''
     Parameters
@@ -838,7 +887,7 @@ def word_tokenize(text, pos_tagged=False):
         else:
             try:
                 if "Can not open" in str(err):
-                    user_folder=get_library_location("PgsFile")+"/PgsFile/Corpora/Stopwords"
+                    user_folder=get_library_location("PgsFile")+"/PgsFile/models"
                     destination_folder=get_library_location("nlpir-python")+"/nlpir/Data"
                     source_file=find_user_files_in_upper_folder(user_folder, "NLPIR")[0]
                     copy_file(source_file, destination_folder)  
@@ -850,6 +899,296 @@ def word_tokenize(text, pos_tagged=False):
 
     return words  
 
+import re
+from abc import ABC, abstractmethod
+from typing import Iterator, List, Tuple
+class TokenizerI(ABC):
+    """
+    A processing interface for tokenizing a string.
+    Subclasses must define ``tokenize()`` or ``tokenize_sents()`` (or both).
+    """
+
+    @abstractmethod
+    def tokenize(self, s: str) -> List[str]:
+        """
+        Return a tokenized copy of *s*.
+
+        :rtype: List[str]
+        """
+        if overridden(self.tokenize_sents):
+            return self.tokenize_sents([s])[0]
+
+    def span_tokenize(self, s: str) -> Iterator[Tuple[int, int]]:
+        """
+        Identify the tokens using integer offsets ``(start_i, end_i)``,
+        where ``s[start_i:end_i]`` is the corresponding token.
+
+        :rtype: Iterator[Tuple[int, int]]
+        """
+        raise NotImplementedError()
+
+    def tokenize_sents(self, strings: List[str]) -> List[List[str]]:
+        """
+        Apply ``self.tokenize()`` to each element of ``strings``.  I.e.:
+
+            return [self.tokenize(s) for s in strings]
+
+        :rtype: List[List[str]]
+        """
+        return [self.tokenize(s) for s in strings]
+
+    def span_tokenize_sents(
+        self, strings: List[str]
+    ) -> Iterator[List[Tuple[int, int]]]:
+        """
+        Apply ``self.span_tokenize()`` to each element of ``strings``.  I.e.:
+
+            return [self.span_tokenize(s) for s in strings]
+
+        :yield: List[Tuple[int, int]]
+        """
+        for s in strings:
+            yield list(self.span_tokenize(s))
+
+class MacIntyreContractions:
+    """
+    List of contractions adapted from Robert MacIntyre's tokenizer.
+    """
+
+    CONTRACTIONS2 = [
+        r"(?i)\b(can)(?#X)(not)\b",
+        r"(?i)\b(d)(?#X)('ye)\b",
+        r"(?i)\b(gim)(?#X)(me)\b",
+        r"(?i)\b(gon)(?#X)(na)\b",
+        r"(?i)\b(got)(?#X)(ta)\b",
+        r"(?i)\b(lem)(?#X)(me)\b",
+        r"(?i)\b(more)(?#X)('n)\b",
+        r"(?i)\b(wan)(?#X)(na)(?=\s)",
+    ]
+    CONTRACTIONS3 = [r"(?i) ('t)(?#X)(is)\b", r"(?i) ('t)(?#X)(was)\b"]
+    CONTRACTIONS4 = [r"(?i)\b(whad)(dd)(ya)\b", r"(?i)\b(wha)(t)(cha)\b"]
+            
+class NLTKWordTokenizer(TokenizerI):
+    """
+    The NLTK tokenizer that has improved upon the TreebankWordTokenizer.
+
+    This is the method that is invoked by ``word_tokenize()``.  It assumes that the
+    text has already been segmented into sentences, e.g. using ``sent_tokenize()``.
+
+    The tokenizer is "destructive" such that the regexes applied will munge the
+    input string to a state beyond re-construction. It is possible to apply
+    `TreebankWordDetokenizer.detokenize` to the tokenized outputs of
+    `NLTKDestructiveWordTokenizer.tokenize` but there's no guarantees to
+    revert to the original string.
+    """
+
+    # Starting quotes.
+    STARTING_QUOTES = [
+        (re.compile("([«“‘„]|[`]+)", re.U), r" \1 "),
+        (re.compile(r"^\""), r"``"),
+        (re.compile(r"(``)"), r" \1 "),
+        (re.compile(r"([ \(\[{<])(\"|\'{2})"), r"\1 `` "),
+        (re.compile(r"(?i)(\')(?!re|ve|ll|m|t|s|d|n)(\w)\b", re.U), r"\1 \2"),
+    ]
+
+    # Ending quotes.
+    ENDING_QUOTES = [
+        (re.compile("([»”’])", re.U), r" \1 "),
+        (re.compile(r"''"), " '' "),
+        (re.compile(r'"'), " '' "),
+        (re.compile(r"\s+"), " "),
+        (re.compile(r"([^' ])('[sS]|'[mM]|'[dD]|') "), r"\1 \2 "),
+        (re.compile(r"([^' ])('ll|'LL|'re|'RE|'ve|'VE|n't|N'T) "), r"\1 \2 "),
+    ]
+
+    # For improvements for starting/closing quotes from TreebankWordTokenizer,
+    # see discussion on https://github.com/nltk/nltk/pull/1437
+    # Adding to TreebankWordTokenizer, nltk.word_tokenize now splits on
+    # - chevron quotes u'\xab' and u'\xbb'
+    # - unicode quotes u'\u2018', u'\u2019', u'\u201c' and u'\u201d'
+    # See https://github.com/nltk/nltk/issues/1995#issuecomment-376741608
+    # Also, behavior of splitting on clitics now follows Stanford CoreNLP
+    # - clitics covered (?!re|ve|ll|m|t|s|d)(\w)\b
+
+    # Punctuation.
+    PUNCTUATION = [
+        (re.compile(r'([^\.])(\.)([\]\)}>"\'' "»”’ " r"]*)\s*$", re.U), r"\1 \2 \3 "),
+        (re.compile(r"([:,])([^\d])"), r" \1 \2"),
+        (re.compile(r"([:,])$"), r" \1 "),
+        (
+            re.compile(r"\.{2,}", re.U),
+            r" \g<0> ",
+        ),  # See https://github.com/nltk/nltk/pull/2322
+        (re.compile(r"[;@#$%&]"), r" \g<0> "),
+        (
+            re.compile(r'([^\.])(\.)([\]\)}>"\']*)\s*$'),
+            r"\1 \2\3 ",
+        ),  # Handles the final period.
+        (re.compile(r"[?!]"), r" \g<0> "),
+        (re.compile(r"([^'])' "), r"\1 ' "),
+        (
+            re.compile(r"[*]", re.U),
+            r" \g<0> ",
+        ),  # See https://github.com/nltk/nltk/pull/2322
+    ]
+
+    # Pads parentheses
+    PARENS_BRACKETS = (re.compile(r"[\]\[\(\)\{\}\<\>]"), r" \g<0> ")
+
+    # Optionally: Convert parentheses, brackets and converts them to PTB symbols.
+    CONVERT_PARENTHESES = [
+        (re.compile(r"\("), "-LRB-"),
+        (re.compile(r"\)"), "-RRB-"),
+        (re.compile(r"\["), "-LSB-"),
+        (re.compile(r"\]"), "-RSB-"),
+        (re.compile(r"\{"), "-LCB-"),
+        (re.compile(r"\}"), "-RCB-"),
+    ]
+
+    DOUBLE_DASHES = (re.compile(r"--"), r" -- ")
+
+    # List of contractions adapted from Robert MacIntyre's tokenizer.
+    _contractions = MacIntyreContractions()
+    CONTRACTIONS2 = list(map(re.compile, _contractions.CONTRACTIONS2))
+    CONTRACTIONS3 = list(map(re.compile, _contractions.CONTRACTIONS3))
+
+    def tokenize(
+        self, text: str, convert_parentheses: bool = False, return_str: bool = False
+    ) -> List[str]:
+        r"""Return a tokenized copy of `text`.
+
+        >>> from nltk.tokenize import NLTKWordTokenizer
+        >>> s = '''Good muffins cost $3.88 (roughly 3,36 euros)\nin New York.  Please buy me\ntwo of them.\nThanks.'''
+        >>> NLTKWordTokenizer().tokenize(s) # doctest: +NORMALIZE_WHITESPACE
+        ['Good', 'muffins', 'cost', '$', '3.88', '(', 'roughly', '3,36',
+        'euros', ')', 'in', 'New', 'York.', 'Please', 'buy', 'me', 'two',
+        'of', 'them.', 'Thanks', '.']
+        >>> NLTKWordTokenizer().tokenize(s, convert_parentheses=True) # doctest: +NORMALIZE_WHITESPACE
+        ['Good', 'muffins', 'cost', '$', '3.88', '-LRB-', 'roughly', '3,36',
+        'euros', '-RRB-', 'in', 'New', 'York.', 'Please', 'buy', 'me', 'two',
+        'of', 'them.', 'Thanks', '.']
+
+
+        :param text: A string with a sentence or sentences.
+        :type text: str
+        :param convert_parentheses: if True, replace parentheses to PTB symbols,
+            e.g. `(` to `-LRB-`. Defaults to False.
+        :type convert_parentheses: bool, optional
+        :param return_str: If True, return tokens as space-separated string,
+            defaults to False.
+        :type return_str: bool, optional
+        :return: List of tokens from `text`.
+        :rtype: List[str]
+        """
+        if return_str:
+            warnings.warn(
+                "Parameter 'return_str' has been deprecated and should no "
+                "longer be used.",
+                category=DeprecationWarning,
+                stacklevel=2,
+            )
+
+        for regexp, substitution in self.STARTING_QUOTES:
+            text = regexp.sub(substitution, text)
+
+        for regexp, substitution in self.PUNCTUATION:
+            text = regexp.sub(substitution, text)
+
+        # Handles parentheses.
+        regexp, substitution = self.PARENS_BRACKETS
+        text = regexp.sub(substitution, text)
+        # Optionally convert parentheses
+        if convert_parentheses:
+            for regexp, substitution in self.CONVERT_PARENTHESES:
+                text = regexp.sub(substitution, text)
+
+        # Handles double dash.
+        regexp, substitution = self.DOUBLE_DASHES
+        text = regexp.sub(substitution, text)
+
+        # add extra space to make things easier
+        text = " " + text + " "
+
+        for regexp, substitution in self.ENDING_QUOTES:
+            text = regexp.sub(substitution, text)
+
+        for regexp in self.CONTRACTIONS2:
+            text = regexp.sub(r" \1 \2 ", text)
+        for regexp in self.CONTRACTIONS3:
+            text = regexp.sub(r" \1 \2 ", text)
+
+        # We are not using CONTRACTIONS4 since
+        # they are also commented out in the SED scripts
+        # for regexp in self._contractions.CONTRACTIONS4:
+        #     text = regexp.sub(r' \1 \2 \3 ', text)
+
+        return text.split()
+
+    def span_tokenize(self, text: str) -> Iterator[Tuple[int, int]]:
+        r"""
+        Returns the spans of the tokens in ``text``.
+        Uses the post-hoc nltk.tokens.align_tokens to return the offset spans.
+
+            >>> from nltk.tokenize import NLTKWordTokenizer
+            >>> s = '''Good muffins cost $3.88\nin New (York).  Please (buy) me\ntwo of them.\n(Thanks).'''
+            >>> expected = [(0, 4), (5, 12), (13, 17), (18, 19), (19, 23),
+            ... (24, 26), (27, 30), (31, 32), (32, 36), (36, 37), (37, 38),
+            ... (40, 46), (47, 48), (48, 51), (51, 52), (53, 55), (56, 59),
+            ... (60, 62), (63, 68), (69, 70), (70, 76), (76, 77), (77, 78)]
+            >>> list(NLTKWordTokenizer().span_tokenize(s)) == expected
+            True
+            >>> expected = ['Good', 'muffins', 'cost', '$', '3.88', 'in',
+            ... 'New', '(', 'York', ')', '.', 'Please', '(', 'buy', ')',
+            ... 'me', 'two', 'of', 'them.', '(', 'Thanks', ')', '.']
+            >>> [s[start:end] for start, end in NLTKWordTokenizer().span_tokenize(s)] == expected
+            True
+
+        :param text: A string with a sentence or sentences.
+        :type text: str
+        :yield: Tuple[int, int]
+        """
+        raw_tokens = self.tokenize(text)
+
+        # Convert converted quotes back to original double quotes
+        # Do this only if original text contains double quote(s) or double
+        # single-quotes (because '' might be transformed to `` if it is
+        # treated as starting quotes).
+        if ('"' in text) or ("''" in text):
+            # Find double quotes and converted quotes
+            matched = [m.group() for m in re.finditer(r"``|'{2}|\"", text)]
+
+            # Replace converted quotes back to double quotes
+            tokens = [
+                matched.pop(0) if tok in ['"', "``", "''"] else tok
+                for tok in raw_tokens
+            ]
+        else:
+            tokens = raw_tokens
+
+        yield from align_tokens(tokens, text)
+
+# Standard word tokenizer.
+_treebank_word_tokenizer = NLTKWordTokenizer()
+def word_tokenize2(text, preserve_line=False):
+    """
+    Return a tokenized copy of *text*,
+    using NLTK's recommended word tokenizer
+    (currently an improved :class:`.TreebankWordTokenizer`
+    along with :class:`.PunktSentenceTokenizer`
+    for the specified language).
+
+    :param text: text to split into words
+    :type text: str
+    :param language: the model name in the Punkt corpus
+    :type language: str
+    :param preserve_line: A flag to decide whether to sentence tokenize the text or not.
+    :type preserve_line: bool
+    """
+    sentences = [text] if preserve_line else sent_tokenize(text)
+    return [
+        token for sent in sentences for token in _treebank_word_tokenizer.tokenize(sent)
+    ]
+        
 def pad_sequence(
     sequence,
     n,
@@ -889,9 +1228,7 @@ def pad_sequence(
         sequence=chain(sequence, (right_pad_symbol,) * (n - 1))
     return sequence
 
-
 # add a flag to pad the sequence so we get peripheral ngrams?
-
 def ngrams(
     sequence,
     n,
@@ -954,7 +1291,6 @@ def ngrams(
         yield tuple(history)
         del history[0]
 
-
 def bigrams(sequence, **kwargs):
     """
     Return the bigrams generated from a sequence of items, as an iterator.
@@ -974,7 +1310,6 @@ def bigrams(sequence, **kwargs):
     for item in ngrams(sequence, 2, **kwargs):
         yield item
 
-
 def trigrams(sequence, **kwargs):
     """
     Return the trigrams generated from a sequence of items, as an iterator.
@@ -994,7 +1329,6 @@ def trigrams(sequence, **kwargs):
     for item in ngrams(sequence, 3, **kwargs):
         yield item
 
-
 def everygrams(sequence, min_len=1, max_len=-1, **kwargs):
     """
     Returns all possible ngrams generated from a sequence of items, as an iterator.
@@ -1148,6 +1482,18 @@ def uninstall_package(package_name: str):
     import pip
     pip.main(['uninstall', package_name, '-y'])
 
+# A list of conda configuration commands.
+conda_mirror_commands=[
+    "pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple", # Windows recommended
+    "conda config --add channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/", # MacOS recommended
+    "conda config --add channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/free/",
+    "conda config --add channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/main/",
+    "conda config --append channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/fastai/",
+    "conda config --append channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/pytorch/",
+    "conda config --append channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/bioconda/",
+    "pip config set global.index-url https://mirrors.aliyun.com/pypi/simple/"
+]
+
 def DirList(root_dir: str) -> tuple: 
     """
     List the contents of a directory and return two lists containing the names of the directories and files in the directory.
@@ -1251,10 +1597,8 @@ def sort_strings_with_embedded_numbers(strings: list) -> list:
     """
     # Sort the strings using the extract_numbers() function as the key
     sorted_strings=sorted(strings, key=extract_numbers)
-
     return sorted_strings
 
-
 def run_command(command: str) -> str:
     """
     Run a command and return its output as a string.
@@ -1444,7 +1788,6 @@ class PGScraper(object):
             return all_want_list   
 
 
-
     
 # -*- coding: utf-8 -*-
 """
@@ -1490,7 +1833,6 @@ class ProgressData(object):
                   self.size, self.unit, progress, speed, self.unit))
             print('%50s'%('/'*int((1-progress)*50)))
 
-
 def levenshtein_distance(s, t):
 	m, n=len(s), len(t)
 	if m < n:
@@ -1516,7 +1858,7 @@ pgs_abbres_words=['A.B.','A.D.','A.G.','A.I.','A.M.','A.P.','A.V.','AFP.','Ala.'
 def clean_text(text): #清洗除了句号以外的其他标点符号问题
     # 在标点符号右边邻接单词前添加空格
     import re
-    text=replace_chinese_punctuation_with_english(text) 
+    # text=replace_chinese_punctuation_with_english(text) 
     text=re.sub(r'(?<=[\?\!\,\;\:\)\]\}])\s*(?=\w)', ' ', text)
     # 删除标点符号与左边单词之间的空格
     text=re.sub(r'\s*([\?\!\,\;\:\)\]\}\>])', r'\1', text)
@@ -1532,24 +1874,45 @@ def clean_text(text): #清洗除了句号以外的其他标点符号问题
 
 def clean_text_with_abbreviations(text):
     import re
-    text=clean_text(text)
-    matches=[]
-    for seg in text.split():
+
+    # 按行分割文本
+    lines = text.splitlines()
+
+    # 清洗每一行
+    cleaned_lines = []
+    for line in lines:
+        cleaned_line = clean_line_with_abbreviations(line)
+        cleaned_lines.append(cleaned_line)
+
+    # 将清洗后的行重新组合成文本
+    cleaned_text = '\n'.join(cleaned_lines)
+    return cleaned_text
+
+def clean_line_with_abbreviations(line):
+    import re
+
+    # 清洗除了句号以外的其他标点符号问题
+    line = clean_text(line)
+
+    matches = []
+    for seg in line.split():
         if "." in seg:
-            if seg.endswith(".") is False:
+            if not seg.endswith("."):
                 matches.append(seg)
             elif seg.endswith("..") and "..." not in seg:
-                text=text.replace("..", ".")
-    
+                line = line.replace("..", ".")
+
     for match in matches:
         if any(word in match for word in pgs_abbres_words):
-            inter=match.split(".")
-            new_match="".join([w+"." for w in inter[0:-1]])+" "+inter[-1]
-            text=text.replace(match, new_match)
+            inter = match.split(".")
+            new_match = "".join([w + "." for w in inter[0:-1]]) + " " + inter[-1]
+            line = line.replace(match, new_match)
         else:
-            text=text.replace(match, match.replace(".",". "))
-    text=re.sub(r'\s+\.', '.', text)
-    return text
+            line = line.replace(match, match.replace(".", ". "))
+
+    line = re.sub(r'\s+\.', '.', line)
+    return line
+
 
 import shutil
 def move_file(source_file, destination_folder, new_file_name=None):
@@ -1597,7 +1960,6 @@ def copy_file(source_file, destination_folder, new_file_name=None):
 
     # Copy the file to the destination folder
     shutil.copy2(source_file, destination_file)
-
     
 def check_empty_cells(file_path):
     """
@@ -1636,7 +1998,6 @@ def makefile(file_path):
     else:
         write_to_txt(file_path, "")
         
-        
 def save_dict_to_excel(data, output_file, headers=None):
     """
     Save Python dictionary data into an Excel .xlsx file with custom headers.
@@ -1845,4 +2206,126 @@ def get_stopwords(language=None):
         return en_stopwords
     else:
         lang_stopwords=get_data_lines(find_txt_files_with_keyword(stopwords_path, language)[0])
-        return lang_stopwords    
+        return lang_stopwords    
+
+from PIL import Image
+def replace_white_with_transparency(input_path, output_path):
+    """
+    This function opens an image, replaces all white pixels with transparent pixels.
+
+    Parameters:
+    input_path (str): The path to the input image file.
+    output_path (str): The path to save the output image file.
+    """
+    # 从RGB（24位）模式转成RGBA（32位）模式
+    img = Image.open(input_path).convert('RGBA')
+    W, L = img.size
+    white_pixel = (0, 0, 0, 0)  # white
+    for h in range(W):
+      for i in range(L):
+        if img.getpixel((h, i)) == white_pixel:
+          img.putpixel((h, i), (255, 255, 255, 0))   # make it transparent
+    img.save(output_path)
+
+def get_font_path(font_name=None):
+    '''
+    Retrieves the file path of a specified font.
+
+    Parameters
+    ----------
+    font_name : str, optional
+        The name of the font file (must end with ".ttf"). If provided, it should match one of the available fonts in the library, such as:
+        - 'DejaVuSans.ttf'
+        - '书体坊赵九江钢笔行书体.ttf'
+        - '全新硬笔楷书简.ttf'
+        - '全新硬笔行书简.ttf'
+        - '博洋行书3500.TTF'
+        - '陆柬之行书字体.ttf'
+        The default is None, which will return the path for 'DejaVuSans.ttf'.
+
+    Returns
+    -------
+    font_path : str
+        The full file path of the specified font. If no font name is provided, the default path for 'DejaVuSans.ttf' will be returned.
+        Example: "C:/Windows/Fonts/simhei.ttf"
+    '''
+    
+    font_folder = get_library_location("PgsFile") + "/PgsFile/models/fonts"
+    if font_name is None:
+        font_path = get_full_path(font_folder, "DejaVuSans.ttf")
+    else:
+        font_path = get_full_path(font_folder, font_name)
+    return font_path
+
+simhei_default_font_path_MacOS_Windows=["/System/Library/Fonts/STHeiti Medium.ttc",
+                   r"C:\Windows\Fonts\simhei.ttf",  # Use a font that supports Chinese characters
+                   ]
+
+
+def get_env_variable(variable_name):
+    # Get the value of the specified environment variable
+    value = os.getenv(variable_name)
+
+    # Check if the environment variable is set
+    if value is not None:
+        print(f"{variable_name} is set to: {value}")
+    else:
+        print(f"{variable_name} is not set.")
+
+import subprocess
+def set_permanent_environment_variable(variable_name, variable_value, system_wide=False):
+    """
+    Sets a permanent environment variable on Windows using the `setx` command.
+
+    Args:
+        variable_name (str): The name of the environment variable.
+        variable_value (str): The value to set for the environment variable.
+        system_wide (bool): If True, sets the variable system-wide (requires admin privileges).
+                            If False, sets the variable for the current user only.
+    """
+    try:
+        # Construct the setx command
+        command = ['setx', variable_name, variable_value]
+        if system_wide:
+            command.append('/M')  # Add /M flag for system-wide variables
+
+        # Run the command
+        subprocess.run(command, shell=True, check=True)
+
+        print(f'Permanent environment variable {variable_name} set to {variable_value} '
+              f'({"system-wide" if system_wide else "user-level"}).')
+    except subprocess.CalledProcessError as e:
+        print(f'Failed to set environment variable: {e}')
+    except Exception as e:
+        print(f'An error occurred: {e}')
+
+def delete_permanent_environment_variable(variable_name, system_wide=False):
+    """
+    Deletes a permanent environment variable on Windows using the `reg` command.
+
+    Args:
+        variable_name (str): The name of the environment variable to delete.
+        system_wide (bool): If True, deletes the variable system-wide (requires admin privileges).
+                            If False, deletes the variable for the current user only.
+    """
+    try:
+        # Determine the registry key based on the scope
+        if system_wide:
+            reg_key = r'HKLM\SYSTEM\CurrentControlSet\Control\Session Manager\Environment'
+        else:
+            reg_key = r'HKCU\Environment'
+
+        # Run the `reg delete` command to remove the variable
+        subprocess.run(
+            ['reg', 'delete', reg_key, '/v', variable_name, '/f'],
+            shell=True,
+            check=True
+        )
+
+        print(f'Permanent environment variable {variable_name} deleted '
+              f'({"system-wide" if system_wide else "user-level"}).')
+    except subprocess.CalledProcessError as e:
+        print(f'Failed to delete environment variable: {e}')
+    except Exception as e:
+        print(f'An error occurred: {e}')
+