PgsFile 0.2.7__py3-none-any.whl → 0.2.8__py3-none-any.whl

PgsFile/PgsFile.py

@@ -2331,3 +2331,244 @@ def delete_permanent_environment_variable(variable_name, system_wide=False):
     except Exception as e:
         print(f'An error occurred: {e}')
 
+
+def calculate_mean_dependency_distance(spacy_doc):
+    """
+    Calculate the mean dependency distance for tokens in a spaCy Doc object.
+
+    The dependency distance is the absolute difference in positions between a token
+    and its syntactic head. This function computes the average of these distances
+    for all tokens in the Doc object, excluding punctuation and the root token.
+
+    Parameters:
+    spacy_doc (spacy.tokens.Doc): The spaCy Doc object to analyze.
+
+    Returns:
+    float: The mean dependency distance. Returns 0 if there are no valid tokens to analyze.
+    """
+    
+    doc=spacy_doc
+    total_distance = 0
+    count = 0
+
+    for token in doc:
+        if token.dep_ not in ("punct", "ROOT"):
+            distance = abs(list(doc).index(token.head) - list(doc).index(token))
+            total_distance += distance
+            count += 1
+
+    if count == 0:
+        return 0
+
+    mean_distance = total_distance / count
+    return mean_distance
+
+def word_lemmatize(spacy_doc):
+    """
+    Lemmatize the words in a spaCy Doc object and return the lemmatized text.
+
+    This function processes each token in the Doc object, replacing it with its lemma
+    unless the lemma is '-PRON-', in which case the original text of the token is used.
+    The resulting lemmatized words are joined into a single string.
+
+    Parameters:
+    spacy_doc (spacy.tokens.Doc): The spaCy Doc object to lemmatize.
+
+    Returns:
+    str: The lemmatized text as a single string.
+    """
+    doc = spacy_doc
+    text = ' '.join([word.lemma_ if word.lemma_ != '-PRON-' else word.text for word in doc])
+    return text
+
+def word_NER(spacy_doc):
+    """
+    Extract Named Entities from a spaCy Doc object.
+
+    This function processes the Doc object to identify and extract named entities,
+    returning a list of tuples where each tuple contains the entity text and its label.
+
+    Parameters:
+    spacy_doc (spacy.tokens.Doc): The spaCy Doc object to analyze.
+
+    Returns:
+    list of tuples: A list of tuples where each tuple contains the entity text and its label.
+    """
+    doc = spacy_doc
+    entities = [(ent.text, ent.label_) for ent in doc.ents]
+    return entities
+
+def word_POS(spacy_doc):
+    """
+    Extract Part-Of-Speech (POS) tags from a spaCy Doc object.
+
+    This function processes the Doc object to identify and extract the POS tags for each token,
+    returning a list of tuples where each tuple contains the token text and its corresponding POS tag.
+
+    Parameters:
+    spacy_doc (spacy.tokens.Doc): The spaCy Doc object to analyze.
+
+    Returns:
+    list of tuples: A list of tuples where each tuple contains the token text and its POS tag.
+    """
+    doc = spacy_doc
+    pos_tags = [(token.text, token.pos_) for token in doc]
+    return pos_tags
+
+def extract_noun_phrases(spacy_doc):
+    """
+    Extract noun phrases from a spaCy Doc object.
+
+    This function processes the Doc object to identify and extract noun phrases,
+    returning a list of strings where each string is a noun phrase.
+
+    Parameters:
+    spacy_doc (spacy.tokens.Doc): The spaCy Doc object to analyze.
+
+    Returns:
+    list of str: A list of noun phrases extracted from the Doc object.
+    """
+    doc = spacy_doc
+    noun_phrases = [chunk.text for chunk in doc.noun_chunks]
+    return noun_phrases
+
+def extract_dependency_relations(spacy_doc):
+    """
+    Extract the dependency relations for each word in a spaCy Doc object.
+
+    This function processes the Doc object to identify and extract the dependency relations
+    for each token, returning a list of tuples where each tuple contains the token text,
+    its dependency relation, and the text of its syntactic head.
+
+    Parameters:
+    spacy_doc (spacy.tokens.Doc): The spaCy Doc object to analyze.
+
+    Returns:
+    list of tuples: A list of tuples where each tuple contains the token text,
+                    its dependency relation, and the text of its syntactic head.
+    """
+    doc = spacy_doc
+    dependency_relations = [(token.text, token.dep_, token.head.text) for token in doc]
+    return dependency_relations
+
+def extract_dependency_relations_full(spacy_doc):
+    """
+    Extract comprehensive dependency relations for each word in a spaCy Doc object.
+
+    This function processes the Doc object to identify and extract detailed dependency relations
+    for each token. It returns a list of tuples where each tuple contains the token text,
+    its lemmatized form, its part-of-speech (POS) tag, its dependency relation, the text of its
+    syntactic head, and a list of its child tokens.
+
+    Parameters:
+    spacy_doc (spacy.tokens.Doc): The spaCy Doc object to analyze.
+
+    Returns:
+    list of tuples: A list of tuples where each tuple contains:
+                    - The token text
+                    - The lemmatized form of the token
+                    - The POS tag of the token
+                    - The dependency relation of the token
+                    - The text of the token's syntactic head
+                    - A list of the text of the token's child tokens
+    """
+    doc = spacy_doc
+    dependency_relations = [(token.text, token.lemma_, token.pos_, token.dep_, token.head.text, [child.text for child in token.children]) for token in doc]
+    return dependency_relations
+
+usua_tag_set = {
+    "A": "General & Abstract Terms",
+    "B": "The Body & the Individual",
+    "C": "Arts & Crafts",
+    "E": "Emotional Actions, States & Processes",
+    "F": "Food & Farming",
+    "G": "Government & the Public Domain",
+    "H": "Architecture, Building, Houses & the Home",
+    "I": "Money & Commerce",
+    "K": "Entertainment, Sports & Games",
+    "L": "Life & Living Things",
+    "M": "Movement, Location, Travel & Transport",
+    "N": "Numbers & Measurement",
+    "O": "Substances, Materials, Objects & Equipment",
+    "P": "Education",
+    "Q": "Linguistic Actions, States & Processes",
+    "S": "Social Actions, States & Processes",
+    "T": "Time",
+    "W": "The World & Our Environment",
+    "X": "Psychological Actions, States & Processes",
+    "Y": "Science & Technology",
+    "Z": "Names & Grammatical Words"
+}
+
+
+def get_CET_dics(name=None):
+    '''
+    Parameters
+    ----------
+    name : TYPE, string: like 'CET-4', 'CET-6', etc.
+        DESCRIPTION. The default is None.
+
+    Returns
+    -------
+    TYPE, list: like ['a', 'an', 'abandon', 'able', 'ability', 'aboard', 'abolish', 'abolition', 'about', 'above', 'abroad', 'absent', 'absence', 'absolute', 'absorb']
+        DESCRIPTION. The default will return a list of English CET (China's College English Test band 4 & 6) words.
+    '''
+    
+    dic_path=get_library_location("PgsFile")+"/PgsFile/models/dics"
+    if name is None:
+        cet_words=get_data_lines(find_txt_files_with_keyword(dic_path, "cet-4")[0])
+        return cet_words
+    else:
+        cet_words=get_data_lines(find_txt_files_with_keyword(dic_path, name)[0])
+        return cet_words   
+
+def get_BNC_dic():
+    '''
+    Returns
+    -------
+    TYPE, pandas dataframe: 
+              List   ... Total frequency
+        0        1k  ...         2525253
+        1        1k  ...           47760
+        2        1k  ...          192168
+        3        1k  ...           25370
+        4        1k  ...            9284
+            ...  ...             ...
+        24997   25k  ...               0
+        24998   25k  ...               0
+        24999   25k  ...               9
+        25000   25k  ...               4
+        25001   25k  ...               9
+        
+        [25002 rows x 4 columns]        
+        DESCRIPTION. The default will return a dataframe of the most commonly used English word list based on the BNC-COCA corpus.
+    '''
+    import pandas as pd
+    inter=get_library_location("PgsFile")+"/PgsFile/models/dics"
+    dic_path=get_full_path(inter, "BNC_COCA_lists.xlsx")
+    print(dic_path)
+    df=pd.read_excel(dic_path)
+    return df
+    
+def get_LLMs_prompt(task=None):
+    '''
+    Parameters
+    ----------
+    task : TYPE, string: like 'MIP', 'WSD', etc.
+        DESCRIPTION. The default is None.
+
+    Returns
+    -------
+    TYPE, string: like LLM Prompt for Metaphor Analysis Task:
+        Identify all metaphorical expressions in the provided text using MIP (Metaphor Identification Procedure).
+        Categorize each metaphor into one of the following CDA sub-types (with brief justification).
+        DESCRIPTION. The default will return a text promt for specific LLM task.
+    '''
+    
+    dic_path=get_library_location("PgsFile")+"/PgsFile/models/prompts"
+    if task is None:
+        user_prompt=get_data_text(find_txt_files_with_keyword(dic_path, "mip")[0])
+        return user_prompt
+    else:
+        user_prompt=get_data_text(find_txt_files_with_keyword(dic_path, task)[0])
+        return user_prompt   

PgsFile/__init__.py

@@ -32,8 +32,8 @@ from .PgsFile import get_env_variable, get_all_env_variables
 
 # 6. Data cleaning
 from .PgsFile import BigPunctuation, StopTags, Special, yhd
-from .PgsFile import ZhStopWords, EnPunctuation, get_stopwords
-from .PgsFile import nltk_en_tags, nltk_tag_mapping, thulac_tags, ICTCLAS2008, LangCodes, pgs_abbres_words
+from .PgsFile import ZhStopWords, EnPunctuation, get_stopwords, get_CET_dics, get_BNC_dic
+from .PgsFile import nltk_en_tags, nltk_tag_mapping, thulac_tags, ICTCLAS2008, LangCodes, pgs_abbres_words, usua_tag_set
 from .PgsFile import check_contain_chinese, check_contain_number
 from .PgsFile import replace_chinese_punctuation_with_english
 from .PgsFile import replace_english_punctuation_with_chinese
@@ -45,11 +45,15 @@ from .PgsFile import strQ2B_raw, strQ2B_words
 from .PgsFile import ngrams, bigrams, trigrams, everygrams, compute_similarity
 from .PgsFile import word_list, batch_word_list
 from .PgsFile import cs, cs1, sent_tokenize, word_tokenize, word_tokenize2
+from .PgsFile import word_lemmatize, word_POS, word_NER
+from .PgsFile import extract_noun_phrases, get_LLMs_prompt
+from .PgsFile import extract_dependency_relations, extract_dependency_relations_full
 
 # 8. Maths
 from .PgsFile import len_rows, check_empty_cells
 from .PgsFile import format_float, decimal_to_percent, Percentage
 from .PgsFile import get_text_length_kb, extract_numbers
+from .PgsFile import calculate_mean_dependency_distance
 
 # 9. Visualization
 from .PgsFile import replace_white_with_transparency