bertrend 0.3.5__tar.gz → 0.3.7__tar.gz

{bertrend-0.3.5 → bertrend-0.3.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: bertrend
-Version: 0.3.5
+Version: 0.3.7
 Summary: BERTrend analyses topic evolution over time using state-of-the-art transformer models: topic modelling and LLM
 License: LICENSE.md
 Keywords: topics,BERTopic,transformers,weak signals,LLM

{bertrend-0.3.5 → bertrend-0.3.7}/bertrend/BERTopicModel.py

@@ -237,6 +237,7 @@ class BERTopicModel:
             Precomputed document embeddings.
         embedding_model : SentenceTransformer or str or None, optional
             Sentence transformer (or associated model name) model for embeddings.
+            NB. Important: if not set, the keywords describing the topics may not be correct (e.g. accents strip); therefore, it is recommended to set this parameter.
         zeroshot_topic_list : list[str] or None, optional
             List of topics used for zeroshot classification.
         zeroshot_min_similarity : float or None, optional

{bertrend-0.3.5 → bertrend-0.3.7}/bertrend/config/bertopic_default_config.toml

@@ -30,7 +30,7 @@ prediction_data = true
 
 # CountVectorizer: https://scikit-learn.org/1.5/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html
 [vectorizer_model]
-ngram_range = [1, 1]
+ngram_range = [1, 2]
 stop_words = true # If true, will check `language` parameter and load associated stopwords file
 min_df = 2
 

{bertrend-0.3.5 → bertrend-0.3.7}/bertrend/demos/demos_utils/embed_documents_component.py

@@ -31,6 +31,9 @@ def display_embed_documents_component():
                     local=False,
                     url=SessionStateManager.get("embedding_service_url"),
                 )
+                SessionStateManager.set(
+                    "embedding_model_name", embedding_service.embedding_model_name
+                )
 
             texts = SessionStateManager.get_dataframe("time_filtered_df")[
                 TEXT_COLUMN

bertrend-0.3.7/bertrend/demos/start_topic_analysis_app.sh

@@ -0,0 +1,14 @@
+#
+# Copyright (c) 2024, RTE (https://www.rte-france.com)
+# See AUTHORS.txt
+# SPDX-License-Identifier: MPL-2.0
+# This file is part of BERTrend.
+#
+
+
+# Set logs directory and create if not exists
+export BERTREND_LOGS_DIR=$BERTREND_BASE_DIR/logs/bertrend
+mkdir -p $BERTREND_LOGS_DIR
+
+echo "Starting BERTrend Topic Analysis demo"
+cd `pwd`/topic_analysis && CUDA_VISIBLE_DEVICES=0 streamlit run app.py 2>&1 | tee -a $BERTREND_LOGS_DIR/topic_analysis_demo.log

bertrend-0.3.7/bertrend/demos/start_weak_signals_app.sh

@@ -0,0 +1,14 @@
+#
+# Copyright (c) 2024, RTE (https://www.rte-france.com)
+# See AUTHORS.txt
+# SPDX-License-Identifier: MPL-2.0
+# This file is part of BERTrend.
+#
+
+
+# Set logs directory and create if not exists
+export BERTREND_LOGS_DIR=$BERTREND_BASE_DIR/logs/bertrend
+mkdir -p $BERTREND_LOGS_DIR
+
+echo "Starting BERTrend Weak Signal Analysis demo"
+cd `pwd`/weak_signals && CUDA_VISIBLE_DEVICES=0 streamlit run app.py 2>&1 | tee -a $BERTREND_LOGS_DIR/weak_signals_demo.log

bertrend-0.3.7/bertrend/demos/topic_analysis/.streamlit/config.toml

@@ -0,0 +1,6 @@
+[theme]
+primaryColor = "royalblue"
+baseFontSize = 13
+
+[server]
+port = 8083

{bertrend-0.3.5 → bertrend-0.3.7}/bertrend/demos/topic_analysis/demo_pages/explore_topics.py

@@ -291,7 +291,9 @@ def _display_topic_description(filtered_df: pd.DataFrame):
     ):
         with st.spinner(translate("generating_description")):
             language_code = (
-                "fr" if SessionStateManager.get("language") == "French" else "en"
+                "fr"
+                if SessionStateManager.get("internationalization_language") == "fr"
+                else "en"
             )
             gpt_description = generate_topic_description(
                 st.session_state["topic_model"],

{bertrend-0.3.5 → bertrend-0.3.7}/bertrend/demos/topic_analysis/demo_pages/newsletters_generation.py

@@ -14,6 +14,7 @@ from bertrend.demos.demos_utils.state_utils import (
     save_widget_state,
     restore_widget_state,
     register_multiple_widget,
+    SessionStateManager,
 )
 from bertrend.services.summary.abstractive_summarizer import AbstractiveSummarizer
 from bertrend.services.summary.chatgpt_summarizer import GPTSummarizer
@@ -48,6 +49,12 @@ def generate_newsletter_wrapper(
         else st.session_state["newsletter_nb_docs"]
     )
 
+    language_code = (
+        "fr"
+        if SessionStateManager.get("internationalization_language") == "fr"
+        else "en"
+    )
+
     return generate_newsletter(
         topic_model=st.session_state["topic_model"],
         df=df,
@@ -60,6 +67,7 @@ def generate_newsletter_wrapper(
             st.session_state["summarizer_classname"]
         ],
         summary_mode=st.session_state["summary_mode"],
+        prompt_language=language_code,
     )
 
 

{bertrend-0.3.5 → bertrend-0.3.7}/bertrend/demos/topic_analysis/demo_pages/training_page.py

@@ -127,6 +127,7 @@ def train_model():
         topic_model_output = topic_model.fit(
             docs=dataset,
             embeddings=embeddings,
+            embedding_model=SessionStateManager.get("embedding_model_name"),
         )
         bertopic = topic_model_output.topic_model
 

bertrend-0.3.7/bertrend/demos/weak_signals/.streamlit/config.toml

@@ -0,0 +1,6 @@
+[theme]
+primaryColor = "royalblue"
+baseFontSize = 13
+
+[server]
+port = 8084

{bertrend-0.3.5 → bertrend-0.3.7}/bertrend/llm_utils/newsletter_features.py

@@ -50,7 +50,7 @@ def generate_newsletter(
     openai_model_name: str = None,
     nb_sentences: int = 3,
 ) -> tuple[str, str, str]:
-    """Generates a newsletters based on a trained BERTopic model.
+    """Generates a newsletter based on a trained BERTopic model.
 
     Args:
         topic_model (BERTopic): trained BERTopic model

{bertrend-0.3.5 → bertrend-0.3.7}/bertrend_apps/data_provider/data_provider.py

@@ -84,14 +84,18 @@ class DataProvider(ABC):
         return results as json lines"""
         articles = []
         for entry in queries_batch:
-            logger.info(f"Processing query: {entry}")
-            articles += self.get_articles(
-                query=entry[0],
-                after=entry[1],
-                before=entry[2],
-                max_results=max_results,
-                language=language,
-            )
+            try:
+                logger.info(f"Processing query: {entry}")
+                articles += self.get_articles(
+                    query=entry[0],
+                    after=entry[1],
+                    before=entry[2],
+                    max_results=max_results,
+                    language=language,
+                )
+            except Exception as e:
+                logger.error("Error occurred with query: " + entry[0] + "\n " + str(e))
+                continue
         # remove duplicates
         articles = [dict(t) for t in {tuple(d.items()) for d in articles}]
         logger.info(f"Collected {len(articles)} articles")

bertrend-0.3.7/bertrend_apps/exploration/curebot/.streamlit/config.toml

@@ -0,0 +1,6 @@
+[theme]
+primaryColor = "royalblue"
+baseFontSize = 13
+
+[server]
+port = 8082

{bertrend-0.3.5 → bertrend-0.3.7}/bertrend_apps/exploration/curebot/app_utils.py

@@ -42,12 +42,19 @@ EMBEDDING_SERVICE = EmbeddingService(local=False)
 
 
 class TopicDescription(BaseModel):
-    """Description of topic during a time period"""
+    """Description of a topic during a time period"""
 
     # Title of the topic
     title: str
 
 
+class TopicSummary(BaseModel):
+    """Summary of a topic"""
+
+    # Title of the topic
+    summary: str
+
+
 @st.cache_data(show_spinner=False)
 def concat_data_from_files(files: list[UploadedFile]) -> pd.DataFrame:
     """
@@ -124,6 +131,7 @@ def fit_bertopic(
     topic_model_output = topic_model.fit(
         docs=docs,
         embeddings=embeddings,
+        embedding_model=EMBEDDING_SERVICE.embedding_model_name,
         zeroshot_topic_list=zeroshot_topic_list,
         zeroshot_min_similarity=0.65,
     )
@@ -237,12 +245,12 @@ def create_newsletter(
             endpoint=LLM_CONFIG["endpoint"],
             model=LLM_CONFIG["model"],
         )
-        response = llm_client.generate(
+        response = llm_client.parse(
             user_prompt=user_prompt,
             system_prompt=TOPIC_SUMMARY_SYSTEM_PROMPT,
-            response_format={"type": "json_object"},
+            response_format=TopicSummary,
         )
-        topic_dict["summary"] = json.loads(response)["résumé"]
+        topic_dict["summary"] = response.summary
         topic_dict["articles"] = []
         for _, row in topic_df.iterrows():
             article_dict = {}

{bertrend-0.3.5 → bertrend-0.3.7}/bertrend_apps/exploration/curebot/prompts.py

@@ -19,6 +19,6 @@ Le résumé doit être concis (maximum 100 mots) et représenter au mieux la sp
 Il doit pas commencer par "Les articles parlent de..." ou équivalent et doit être écrit dans un style journalistique.
 Répondez sous la forme d'un JSON suivant le format ci-dessous :
 {
-    "résumé": "<votre résumé du thème>"
+    "summary": "<votre résumé du thème>"
 }
 """

{bertrend-0.3.5 → bertrend-0.3.7}/bertrend_apps/exploration/curebot/start_newsletter_generator.sh

@@ -5,4 +5,9 @@
 # This file is part of BERTrend.
 #
 
-OPENAI_API_KEY=$OPENAI_API_KEY_VEILLE CUDA_VISIBLE_DEVICES=0 streamlit run  --theme.primaryColor royalblue --server.port 8686 app.py
+
+# Set logs directory and create if not exists
+export BERTREND_LOGS_DIR=$BERTREND_BASE_DIR/logs/bertrend
+mkdir -p $BERTREND_LOGS_DIR
+
+OPENAI_API_KEY=$OPENAI_API_KEY_VEILLE CUDA_VISIBLE_DEVICES=0 streamlit run app.py | tee -a $BERTREND_LOGS_DIR/curebot_demo.log

{bertrend-0.3.5 → bertrend-0.3.7}/bertrend_apps/exploration/curebot/tabs/tab1.py

@@ -130,9 +130,9 @@ def train_model() -> None:
 
     # Train topic model
     bertopic, topics = fit_bertopic(
-        texts_list,
-        embeddings,
-        st.session_state["min_articles_per_topic"],
+        docs=texts_list,
+        embeddings=embeddings,
+        min_articles_per_topic=st.session_state["min_articles_per_topic"],
         zeroshot_topic_list=zeroshot_topic_list,
     )
 

{bertrend-0.3.5 → bertrend-0.3.7}/bertrend_apps/newsletters/__main__.py

@@ -216,6 +216,7 @@ if __name__ == "__main__":
         output = topic_model.fit(
             docs=dataset[TEXT_COLUMN],
             embeddings=embeddings,
+            embedding_model=embedding_model,
         )
         return output.topics, output.topic_model
 

bertrend-0.3.7/bertrend_apps/prospective_demo/.streamlit/config.toml

@@ -0,0 +1,6 @@
+[theme]
+primaryColor = "royalblue"
+baseFontSize = 13
+
+[server]
+port = 8081

bertrend-0.3.7/bertrend_apps/prospective_demo/start_demo.sh

@@ -0,0 +1,16 @@
+#
+# Copyright (c) 2024, RTE (https://www.rte-france.com)
+# See AUTHORS.txt
+# SPDX-License-Identifier: MPL-2.0
+# This file is part of BERTrend.
+#
+
+
+
+
+# Set logs directory and create if not exists
+export BERTREND_LOGS_DIR=$BERTREND_BASE_DIR/logs/bertrend
+mkdir -p $BERTREND_LOGS_DIR
+
+echo "Starting BERTrend Prospective Analysis demo"
+CUDA_VISIBLE_DEVICES=0 streamlit run app.py 2>&1 | tee -a $BERTREND_LOGS_DIR/prospective_analysis_demo.log

{bertrend-0.3.5 → bertrend-0.3.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "bertrend"
-version = "0.3.5"
+version = "0.3.7"
 description = "BERTrend analyses topic evolution over time using state-of-the-art transformer models: topic modelling and LLM"
 license = "LICENSE.md"
 readme = "README.md"

bertrend-0.3.5/bertrend/demos/start_topic_analysis_app.sh

@@ -1,9 +0,0 @@
-#
-# Copyright (c) 2024, RTE (https://www.rte-france.com)
-# See AUTHORS.txt
-# SPDX-License-Identifier: MPL-2.0
-# This file is part of BERTrend.
-#
-
-# Starts the Topic Analysis application
-CUDA_VISIBLE_DEVICES=0 streamlit run --theme.primaryColor royalblue topic_analysis/app.py

bertrend-0.3.5/bertrend/demos/start_weak_signals_app.sh

@@ -1,9 +0,0 @@
-#
-# Copyright (c) 2024, RTE (https://www.rte-france.com)
-# See AUTHORS.txt
-# SPDX-License-Identifier: MPL-2.0
-# This file is part of BERTrend.
-#
-
-# Starts the Trend Analysis application
-CUDA_VISIBLE_DEVICES=0 streamlit run --theme.primaryColor royalblue weak_signals/app.py

bertrend-0.3.5/bertrend_apps/prospective_demo/.streamlit/config.toml

@@ -1,3 +0,0 @@
-[theme]
-primaryColor = "royalblue"
-baseFontSize = 13

bertrend-0.3.5/bertrend_apps/prospective_demo/start_demo.sh

@@ -1,9 +0,0 @@
-#
-# Copyright (c) 2024, RTE (https://www.rte-france.com)
-# See AUTHORS.txt
-# SPDX-License-Identifier: MPL-2.0
-# This file is part of BERTrend.
-#
-
-# Starts the Trend Analysis application
-CUDA_VISIBLE_DEVICES=0 streamlit run app.py