opsci-toolbox 0.0.10__py3-none-any.whl → 0.0.11__py3-none-any.whl

opsci_toolbox/helpers/nlp.py

@@ -48,6 +48,28 @@ def remove_html_tags(text: str) -> str:
     soup = BeautifulSoup(text, "html.parser")
     return soup.get_text()
 
+def remove_rt(text: str) -> str:
+    """
+    Remove the retweet tag from a given text.
+
+    Args:
+    - text (str): The input text possibly containing a retweet tag in the format "RT @username: ".
+
+    Returns:
+    - str: The cleaned text with the retweet tag removed.
+
+    Example:
+    >>> remove_rt("RT @user123: Check out this tweet!")
+    'Check out this tweet!'
+    """
+    # Regular expression pattern to match "RT @username: "
+    pattern = r'RT @\w+: '
+
+    # Substitute the pattern with an empty string
+    cleaned_text = re.sub(pattern, '', text)
+
+    return cleaned_text
+
 def filter_by_query(df: pd.DataFrame, col_text: str, query: str, ignore_case: bool = True, ignore_accent: bool = True, match_word: bool = False) -> pd.DataFrame:
     """
     Filter DataFrame rows by a query on a specific text column.
@@ -91,6 +113,7 @@ def TM_clean_text(df: pd.DataFrame, col: str, col_clean: str) -> pd.DataFrame:
         df : pandas DataFrame
             The DataFrame with cleaned text data.
     """
+    df[col_clean] = df[col].apply(remove_rt)
     df[col_clean] = df[col].apply(lambda x : urls(x, repl= ''))
     df[col_clean] = df.apply(lambda row: " ".join(filter(lambda x: x[0] != "@", row[col_clean].split())), 1)
     df[col_clean] = df[col_clean].apply(remove_extra_spaces)
@@ -1358,14 +1381,14 @@ def split_n_sentences(nlp, df: pd.DataFrame, col_text: str, n_sentences: int = 1
     return df
 
 
-def spacy_NER(nlp, df: pd.DataFrame, col_text: str, entities_to_keep: list = ['PERSON','ORG'], explode: bool = True) -> pd.DataFrame:
+def spacy_NER(nlp, df: pd.DataFrame, col_text: str, entities_to_keep: list = ['PERSON','ORG'], explode: bool = True, batch_size : int = 100, n_process: int =1) -> pd.DataFrame:
     """
     Spacy implementation of NER. 
     To define entities type to keep, call get_labels(nlp, pipe_step="ner", explanations=False)
     explode = False means it returns 1 list of entities per document
     explode = True means it returns 1 entity per row
     
-    Parameters:
+    Args:
         nlp : spacy.language.Language
             The spaCy language processing pipeline.
         df : pd.DataFrame
@@ -1376,6 +1399,10 @@ def spacy_NER(nlp, df: pd.DataFrame, col_text: str, entities_to_keep: list = ['P
             List of entity types to keep. Default is ['PERSON','ORG'].
         explode : bool, optional
             Flag indicating whether to explode the DataFrame to have one entity per row. Default is True.
+        batch_size : int, optional
+            Batch sizes
+        n_process : int, optional
+            Number of processes
     
     Returns:
         pd.DataFrame
@@ -1385,43 +1412,40 @@ def spacy_NER(nlp, df: pd.DataFrame, col_text: str, entities_to_keep: list = ['P
         This function performs Named Entity Recognition (NER) using spaCy on a DataFrame with text data. It extracts entities of the specified types 
         and stores the NER information in separate columns. If 'explode' is set to True, it returns one entity per row in the DataFrame.
     """
-    # Create columns to store the NER information
-    df['NER_type'] = None
-    df['NER_text'] = None
-    df['NER_start_char'] = None
-    df['NER_end_char'] = None
-
-    # Function to process each row in the DataFrame
-    def process_row(row):
-        doc = nlp(row[col_text])
-        entities_data = []
+    l_text = df[col_text].tolist()
+    all_records = []
+    for doc in tqdm(nlp.pipe(l_text, batch_size=batch_size, n_process=n_process), total= len(l_text), desc = "NLP Process"):
+        NER_type = []
+        NER_text = []
+        NER_start_char = []
+        NER_end_char=[]
+        # entities_data = []
 
         if len(entities_to_keep)>0:
             for ent in doc.ents:
                 if ent.label_ in entities_to_keep:
-                    entities_data.append([ent.label_, ent.text, ent.start_char, ent.end_char])
+                    NER_type.append(ent.label_)
+                    NER_text.append(ent.text)
+                    NER_start_char.append(ent.start_char)
+                    NER_end_char.append(ent.end_char)
+                    # entities_data.append([ent.label_, ent.text, ent.start_char, ent.end_char])
         else:
             for ent in doc.ents:
-                entities_data.append([ent.label_, ent.text, ent.start_char, ent.end_char])
+                NER_type.append(ent.label_)
+                NER_text.append(ent.text)
+                NER_start_char.append(ent.start_char)
+                NER_end_char.append(ent.end_char)
+                # entities_data.append([ent.label_, ent.text, ent.start_char, ent.end_char])
+        record = (NER_type, NER_text, NER_start_char, NER_end_char)
+        all_records.append(record)
 
-        if entities_data:
-            entity_label, entity_text, start_char, end_char = zip(*entities_data)
-            row['NER_type'] = entity_label
-            row['NER_text'] = entity_text
-            row['NER_start_char'] = start_char
-            row['NER_end_char'] = end_char
-
-        return row
-
-    # Apply the processing function to each row
-    df = df.apply(process_row, axis=1)
+    df[['NER_type', 'NER_text','NER_start_char','NER_end_char']] = pd.DataFrame(all_records, index=df.index)
 
     if explode:
         df= df.explode(['NER_type', 'NER_text','NER_start_char','NER_end_char'])
 
     return df
 
-
 def tokenize(nlp, df: pd.DataFrame, col_text: str, col_tokens: str, pos_to_keep: list, stopwords: list, batch_size: int = 100, n_process: int = 1, stats: bool = True) -> pd.DataFrame:
     """ 
     Spacy implementation to tokenize text
@@ -1901,15 +1925,13 @@ def agglomerative_clustering(embeddings, n_clusters=15, metric="euclidean", link
     
     
     
-def hdbscan_clustering(embeddings, algorithm='best', alpha=1.0, cluster_selection_epsilon=0.0, approx_min_span_tree=True,
-                       gen_min_span_tree=True, leaf_size=40, metric='euclidean', min_cluster_size=5, min_samples=None,
-                       p=None, cluster_selection_method='eom', prediction_data = True):
-    
+def hdbscan_clustering(embeddings, algorithm='best', alpha=1.0, cluster_selection_epsilon=0.0, approx_min_span_tree=True, gen_min_span_tree=True, leaf_size=40, metric='euclidean', min_cluster_size=5, min_samples=None, p=None, cluster_selection_method='eom', prediction_data = True):
     """
     This function performs clustering using the HDBSCAN (Hierarchical Density-Based Spatial Clustering of Applications with Noise) algorithm. It clusters the input data based on the specified parameters and returns the clusterer object, cluster labels for each point, and the probability of each sample being an outlier.
+    
     Args
         embeddings : array-like or sparse matrix, shape (n_samples, n_features). The input data to be clustered.
-        algorithm : {'best', 'generic', 'prims_kdtree', 'boruvka_kdtree', 'boruvka_balltree', 'prims_balltree'}, optional. The algorithm to use for computation. Default is 'best'.
+        algorithm : {'best', 'generic', 'prims_kdtree', 'boruvka_kdtree', 'boruvka_balltree', 'prims_balltree'}, optional. The algorithm to use for computation. Default is best.
         alpha : float, optional. Scaling factor determining the individual weight of the (unnormalized) density estimate. Default is 1.0.
         cluster_selection_epsilon : float, optional. The epsilon value to specify a minimum cluster size. Default is 0.0.
         approx_min_span_tree : bool, optional. Whether to compute an approximation of the minimum spanning tree. Default is True.
@@ -2054,7 +2076,6 @@ def HF_sentiment_classifier(tokenizer, model, text, col_text, filename, dir_json
             proba = torch.sigmoid(model(**inputs).logits).cpu().numpy()[0]
             label = model.config.id2label[proba.argmax()]
             results = {"label":label, "score" : float(proba.max()), col_text : text}
-            print(results)
             write_json(results, dir_json , str(filename))
             
     return results

opsci_toolbox-0.0.11.dist-info/METADATA

@@ -0,0 +1,53 @@
+Metadata-Version: 2.1
+Name: opsci-toolbox
+Version: 0.0.11
+Summary: a complete toolbox
+Home-page: UNKNOWN
+Author: Erwan Le Nagard
+Author-email: erwan@opsci.ai
+License: MIT
+Platform: UNKNOWN
+Requires-Dist: requests <3,>=2.31.0
+Requires-Dist: beautifulsoup4 ==4.9.3
+Requires-Dist: chardet >=4.0.0
+Requires-Dist: chart-studio ==1.1.0
+Requires-Dist: eldar ==0.0.8
+Requires-Dist: emoji ==2.10.1
+Requires-Dist: fa2-modified ==0.3.10
+Requires-Dist: google-api-python-client ==2.122.0
+Requires-Dist: gspread ==6.1.2
+Requires-Dist: hdbscan ==0.8.33
+Requires-Dist: jusText ==3.0.0
+Requires-Dist: langchain ==0.1.20
+Requires-Dist: matplotlib >=3.9.0
+Requires-Dist: mysql-connector-python >=9.0.0
+Requires-Dist: networkx ==3.2.1
+Requires-Dist: nltk ==3.8.1
+Requires-Dist: numpy <1.25.0,>=1.21.5
+Requires-Dist: opencv-python-headless ==4.9.0.80
+Requires-Dist: openpyxl ==3.1.3
+Requires-Dist: pandas >=1.5.3
+Requires-Dist: Pillow >=9.0.1
+Requires-Dist: plotly ==5.19.0
+Requires-Dist: protobuf ==4.23.4
+Requires-Dist: pyarrow >=14.0.2
+Requires-Dist: python-louvain ==0.16
+Requires-Dist: scikit-learn ==1.4.1.post1
+Requires-Dist: scipy <2.0.0,>=1.8.0
+Requires-Dist: sentence-transformers ==2.5.1
+Requires-Dist: setuptools ==59.6.0
+Requires-Dist: spacy ==3.7.4
+Requires-Dist: spacy-language-detection ==0.2.1
+Requires-Dist: spacymoji ==3.1.0
+Requires-Dist: supervision ==0.21.0
+Requires-Dist: textacy ==0.13.0
+Requires-Dist: torch ==2.0.1
+Requires-Dist: tqdm >=4.66.2
+Requires-Dist: trafilatura ==1.7.0
+Requires-Dist: transformers ==4.38.2
+Requires-Dist: umap-learn ==0.5.5
+Requires-Dist: urlextract ==1.9.0
+Requires-Dist: wordcloud ==1.9.3
+
+UNKNOWN
+

{opsci_toolbox-0.0.10.dist-info → opsci_toolbox-0.0.11.dist-info}/RECORD

@@ -4,11 +4,11 @@ opsci_toolbox/apis/rapidapi_helpers.py,sha256=k_hYcRNww5noNkX7zyz5Htggxb15BPoKSl
 opsci_toolbox/apis/webscraping.py,sha256=Gz3hOfhOHUpwHU1Pzj3mB2WdBAcKa2WisYBHMi3lcVE,18343
 opsci_toolbox/apis/youtube_helpers.py,sha256=j4hwCS2BEWRJjd9Q5XBN9FeCrL3lqteyz5dqbtfypdo,17418
 opsci_toolbox/helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-opsci_toolbox/helpers/common.py,sha256=lemGhNwWIxaMwo-X7UsksUMGLV-IOuX_XwC82a50GD4,44672
+opsci_toolbox/helpers/common.py,sha256=nqg9wzgU5DxVTCxEb5LSw2lUnp0f_hKF_Q-DhpRtu6g,45158
 opsci_toolbox/helpers/cv.py,sha256=N3hnLX223UQbdw_YEdUYj10xUXT_95O6BpQt6TbAE08,21092
-opsci_toolbox/helpers/dataviz.py,sha256=IfHByNWAU2rErZMfs3LuwZwJApLN5w320JEbBPuVp6U,115856
+opsci_toolbox/helpers/dataviz.py,sha256=1cIGb-u81cD5iSIkkkrzyrBnfim7fbhm0x_CguHUbf0,202128
 opsci_toolbox/helpers/dates.py,sha256=Wf7HxaUY62IRrY3XPdRIuoaMbGi3QqWf-vStqbRRY_o,2633
-opsci_toolbox/helpers/nlp.py,sha256=r4o7V9tJrj3xt34O_4hN0szbSB4RmveP8qmwCqHOxEY,87988
+opsci_toolbox/helpers/nlp.py,sha256=baq4BsSgeLBgToPOU5RTmDA80dFJwH9xf0jppuAVseU,88947
 opsci_toolbox/helpers/nlp_cuml.py,sha256=XzBfoFMpVIehpRbp60E4wGokpoqJP0lJxs1plOxQqBY,28882
 opsci_toolbox/helpers/sna.py,sha256=XL1BZ-x83xWRNbGsvh7-m8Mdy6iOrWx8vjgaL2_TSmo,31905
 opsci_toolbox/helpers/sql.py,sha256=LMrDWcv1QpfE8HyyrqiKuhhkt930lvME3-AKU89LF38,1928
@@ -16,7 +16,7 @@ opsci_toolbox/helpers/surreaction.py,sha256=JjVvHs7Sf9IJxX0QdHpQ_3E8-c_OS6q_bfUK
 opsci_toolbox/lexicons/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 opsci_toolbox/lexicons/stop_words_en.csv,sha256=4lzjBZHCn_b3lg_CUNkmA_MDQ7DLEpS83k6-dWpkC2o,1957
 opsci_toolbox/lexicons/stop_words_fr.csv,sha256=sPdA8VmyNYbiHg-M8O3tg7ayHvCE3GDg6cF-oSZxICM,6776
-opsci_toolbox-0.0.10.dist-info/METADATA,sha256=DAYpwkedg6Tf4p_JS0ntxq9qUBx9hxWagStKN972RoU,1717
-opsci_toolbox-0.0.10.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
-opsci_toolbox-0.0.10.dist-info/top_level.txt,sha256=fUiqxou4FPec_tOfauTLCKAuepeYLfRyhedycWxVnq4,14
-opsci_toolbox-0.0.10.dist-info/RECORD,,
+opsci_toolbox-0.0.11.dist-info/METADATA,sha256=5h-cfwhi31VKlzrOfdAeZuoKTLB1iyDIA4qqsz-bZGQ,1633
+opsci_toolbox-0.0.11.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+opsci_toolbox-0.0.11.dist-info/top_level.txt,sha256=fUiqxou4FPec_tOfauTLCKAuepeYLfRyhedycWxVnq4,14
+opsci_toolbox-0.0.11.dist-info/RECORD,,

{opsci_toolbox-0.0.10.dist-info → opsci_toolbox-0.0.11.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.37.1)
+Generator: bdist_wheel (0.43.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

opsci_toolbox-0.0.10.dist-info/METADATA

@@ -1,53 +0,0 @@
-Metadata-Version: 2.1
-Name: opsci-toolbox
-Version: 0.0.10
-Summary: a complete toolbox
-Home-page: UNKNOWN
-Author: Erwan Le Nagard
-Author-email: erwan@opsci.ai
-License: MIT
-Platform: UNKNOWN
-Requires-Dist: Pillow (>=9.0.1)
-Requires-Dist: Requests (==2.32.3)
-Requires-Dist: beautifulsoup4 (==4.10.0)
-Requires-Dist: chardet (>=4.0.0)
-Requires-Dist: chart-studio (==1.1.0)
-Requires-Dist: eldar (==0.0.8)
-Requires-Dist: emoji (==2.10.1)
-Requires-Dist: fa2-modified (==0.3.10)
-Requires-Dist: google-api-python-client (==2.122.0)
-Requires-Dist: gspread (==6.1.2)
-Requires-Dist: hdbscan (==0.8.33)
-Requires-Dist: jusText (==3.0.0)
-Requires-Dist: langchain (==0.1.20)
-Requires-Dist: matplotlib (>=3.9.0)
-Requires-Dist: mysql-connector-repackaged (==0.3.1)
-Requires-Dist: networkx (==3.2.1)
-Requires-Dist: nltk (==3.8.1)
-Requires-Dist: numpy (<1.25.0,>=1.21.5)
-Requires-Dist: opencv-python-headless (==4.9.0.80)
-Requires-Dist: openpyxl (==3.1.3)
-Requires-Dist: pandas (>=1.5.3)
-Requires-Dist: plotly (==5.19.0)
-Requires-Dist: protobuf (==4.23.4)
-Requires-Dist: pyarrow (>=14.0.2)
-Requires-Dist: python-louvain (==0.16)
-Requires-Dist: scikit-learn (==1.4.1.post1)
-Requires-Dist: scipy (<2.0.0,>=1.8.0)
-Requires-Dist: sentence-transformers (==2.5.1)
-Requires-Dist: setuptools (==59.6.0)
-Requires-Dist: spacy (==3.7.4)
-Requires-Dist: spacy-language-detection (==0.2.1)
-Requires-Dist: spacymoji (==3.1.0)
-Requires-Dist: supervision (==0.21.0)
-Requires-Dist: textacy (==0.13.0)
-Requires-Dist: torch (==2.0.1)
-Requires-Dist: tqdm (==4.66.2)
-Requires-Dist: trafilatura (==1.7.0)
-Requires-Dist: transformers (==4.38.2)
-Requires-Dist: umap-learn (==0.5.5)
-Requires-Dist: urlextract (==1.9.0)
-Requires-Dist: wordcloud (==1.9.3)
-
-UNKNOWN
-