bertrend 0.3.5__tar.gz → 0.3.6__tar.gz

{bertrend-0.3.5 → bertrend-0.3.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: bertrend
-Version: 0.3.5
+Version: 0.3.6
 Summary: BERTrend analyses topic evolution over time using state-of-the-art transformer models: topic modelling and LLM
 License: LICENSE.md
 Keywords: topics,BERTopic,transformers,weak signals,LLM

{bertrend-0.3.5 → bertrend-0.3.6}/bertrend/config/bertopic_default_config.toml

@@ -30,7 +30,7 @@ prediction_data = true
 
 # CountVectorizer: https://scikit-learn.org/1.5/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html
 [vectorizer_model]
-ngram_range = [1, 1]
+ngram_range = [1, 2]
 stop_words = true # If true, will check `language` parameter and load associated stopwords file
 min_df = 2
 

bertrend-0.3.6/bertrend/demos/start_topic_analysis_app.sh

@@ -0,0 +1,14 @@
+#
+# Copyright (c) 2024, RTE (https://www.rte-france.com)
+# See AUTHORS.txt
+# SPDX-License-Identifier: MPL-2.0
+# This file is part of BERTrend.
+#
+
+
+# Set logs directory and create if not exists
+export BERTREND_LOGS_DIR=$BERTREND_BASE_DIR/logs/bertrend
+mkdir -p $BERTREND_LOGS_DIR
+
+echo "Starting BERTrend Topic Analysis demo"
+cd `pwd`/topic_analysis && CUDA_VISIBLE_DEVICES=0 streamlit run app.py 2>&1 | tee -a $BERTREND_LOGS_DIR/topic_analysis_demo.log

bertrend-0.3.6/bertrend/demos/start_weak_signals_app.sh

@@ -0,0 +1,14 @@
+#
+# Copyright (c) 2024, RTE (https://www.rte-france.com)
+# See AUTHORS.txt
+# SPDX-License-Identifier: MPL-2.0
+# This file is part of BERTrend.
+#
+
+
+# Set logs directory and create if not exists
+export BERTREND_LOGS_DIR=$BERTREND_BASE_DIR/logs/bertrend
+mkdir -p $BERTREND_LOGS_DIR
+
+echo "Starting BERTrend Weak Signal Analysis demo"
+cd `pwd`/weak_signals && CUDA_VISIBLE_DEVICES=0 streamlit run app.py 2>&1 | tee -a $BERTREND_LOGS_DIR/weak_signals_demo.log

bertrend-0.3.6/bertrend/demos/topic_analysis/.streamlit/config.toml

@@ -0,0 +1,6 @@
+[theme]
+primaryColor = "royalblue"
+baseFontSize = 13
+
+[server]
+port = 8083

{bertrend-0.3.5 → bertrend-0.3.6}/bertrend/demos/topic_analysis/demo_pages/explore_topics.py

@@ -291,7 +291,9 @@ def _display_topic_description(filtered_df: pd.DataFrame):
     ):
         with st.spinner(translate("generating_description")):
             language_code = (
-                "fr" if SessionStateManager.get("language") == "French" else "en"
+                "fr"
+                if SessionStateManager.get("internationalization_language") == "French"
+                else "en"
             )
             gpt_description = generate_topic_description(
                 st.session_state["topic_model"],

{bertrend-0.3.5 → bertrend-0.3.6}/bertrend/demos/topic_analysis/demo_pages/newsletters_generation.py

@@ -14,6 +14,7 @@ from bertrend.demos.demos_utils.state_utils import (
     save_widget_state,
     restore_widget_state,
     register_multiple_widget,
+    SessionStateManager,
 )
 from bertrend.services.summary.abstractive_summarizer import AbstractiveSummarizer
 from bertrend.services.summary.chatgpt_summarizer import GPTSummarizer
@@ -48,6 +49,12 @@ def generate_newsletter_wrapper(
         else st.session_state["newsletter_nb_docs"]
     )
 
+    language_code = (
+        "fr"
+        if SessionStateManager.get("internationalization_language") == "French"
+        else "en"
+    )
+
     return generate_newsletter(
         topic_model=st.session_state["topic_model"],
         df=df,
@@ -60,6 +67,7 @@ def generate_newsletter_wrapper(
             st.session_state["summarizer_classname"]
         ],
         summary_mode=st.session_state["summary_mode"],
+        prompt_language=language_code,
     )
 
 

bertrend-0.3.6/bertrend/demos/weak_signals/.streamlit/config.toml

@@ -0,0 +1,6 @@
+[theme]
+primaryColor = "royalblue"
+baseFontSize = 13
+
+[server]
+port = 8084

{bertrend-0.3.5 → bertrend-0.3.6}/bertrend_apps/data_provider/data_provider.py

@@ -84,14 +84,18 @@ class DataProvider(ABC):
         return results as json lines"""
         articles = []
         for entry in queries_batch:
-            logger.info(f"Processing query: {entry}")
-            articles += self.get_articles(
-                query=entry[0],
-                after=entry[1],
-                before=entry[2],
-                max_results=max_results,
-                language=language,
-            )
+            try:
+                logger.info(f"Processing query: {entry}")
+                articles += self.get_articles(
+                    query=entry[0],
+                    after=entry[1],
+                    before=entry[2],
+                    max_results=max_results,
+                    language=language,
+                )
+            except Exception as e:
+                logger.error("Error occurred with query: " + entry[0] + "\n " + str(e))
+                continue
         # remove duplicates
         articles = [dict(t) for t in {tuple(d.items()) for d in articles}]
         logger.info(f"Collected {len(articles)} articles")

bertrend-0.3.6/bertrend_apps/exploration/curebot/.streamlit/config.toml

@@ -0,0 +1,6 @@
+[theme]
+primaryColor = "royalblue"
+baseFontSize = 13
+
+[server]
+port = 8082

{bertrend-0.3.5 → bertrend-0.3.6}/bertrend_apps/exploration/curebot/app_utils.py

@@ -42,12 +42,19 @@ EMBEDDING_SERVICE = EmbeddingService(local=False)
 
 
 class TopicDescription(BaseModel):
-    """Description of topic during a time period"""
+    """Description of a topic during a time period"""
 
     # Title of the topic
     title: str
 
 
+class TopicSummary(BaseModel):
+    """Summary of a topic"""
+
+    # Title of the topic
+    summary: str
+
+
 @st.cache_data(show_spinner=False)
 def concat_data_from_files(files: list[UploadedFile]) -> pd.DataFrame:
     """
@@ -237,12 +244,12 @@ def create_newsletter(
             endpoint=LLM_CONFIG["endpoint"],
             model=LLM_CONFIG["model"],
         )
-        response = llm_client.generate(
+        response = llm_client.parse(
             user_prompt=user_prompt,
             system_prompt=TOPIC_SUMMARY_SYSTEM_PROMPT,
-            response_format={"type": "json_object"},
+            response_format=TopicSummary,
         )
-        topic_dict["summary"] = json.loads(response)["résumé"]
+        topic_dict["summary"] = response.summary
         topic_dict["articles"] = []
         for _, row in topic_df.iterrows():
             article_dict = {}

{bertrend-0.3.5 → bertrend-0.3.6}/bertrend_apps/exploration/curebot/prompts.py

@@ -19,6 +19,6 @@ Le résumé doit être concis (maximum 100 mots) et représenter au mieux la sp
 Il doit pas commencer par "Les articles parlent de..." ou équivalent et doit être écrit dans un style journalistique.
 Répondez sous la forme d'un JSON suivant le format ci-dessous :
 {
-    "résumé": "<votre résumé du thème>"
+    "summary": "<votre résumé du thème>"
 }
 """

{bertrend-0.3.5 → bertrend-0.3.6}/bertrend_apps/exploration/curebot/start_newsletter_generator.sh

@@ -5,4 +5,9 @@
 # This file is part of BERTrend.
 #
 
-OPENAI_API_KEY=$OPENAI_API_KEY_VEILLE CUDA_VISIBLE_DEVICES=0 streamlit run  --theme.primaryColor royalblue --server.port 8686 app.py
+
+# Set logs directory and create if not exists
+export BERTREND_LOGS_DIR=$BERTREND_BASE_DIR/logs/bertrend
+mkdir -p $BERTREND_LOGS_DIR
+
+OPENAI_API_KEY=$OPENAI_API_KEY_VEILLE CUDA_VISIBLE_DEVICES=0 streamlit run app.py | tee -a $BERTREND_LOGS_DIR/curebot_demo.log

bertrend-0.3.6/bertrend_apps/prospective_demo/.streamlit/config.toml

@@ -0,0 +1,6 @@
+[theme]
+primaryColor = "royalblue"
+baseFontSize = 13
+
+[server]
+port = 8081

bertrend-0.3.6/bertrend_apps/prospective_demo/start_demo.sh

@@ -0,0 +1,16 @@
+#
+# Copyright (c) 2024, RTE (https://www.rte-france.com)
+# See AUTHORS.txt
+# SPDX-License-Identifier: MPL-2.0
+# This file is part of BERTrend.
+#
+
+
+
+
+# Set logs directory and create if not exists
+export BERTREND_LOGS_DIR=$BERTREND_BASE_DIR/logs/bertrend
+mkdir -p $BERTREND_LOGS_DIR
+
+echo "Starting BERTrend Prospective Analysis demo"
+CUDA_VISIBLE_DEVICES=0 streamlit run app.py 2>&1 | tee -a $BERTREND_LOGS_DIR/prospective_analysis_demo.log

{bertrend-0.3.5 → bertrend-0.3.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "bertrend"
-version = "0.3.5"
+version = "0.3.6"
 description = "BERTrend analyses topic evolution over time using state-of-the-art transformer models: topic modelling and LLM"
 license = "LICENSE.md"
 readme = "README.md"

bertrend-0.3.5/bertrend/demos/start_topic_analysis_app.sh

@@ -1,9 +0,0 @@
-#
-# Copyright (c) 2024, RTE (https://www.rte-france.com)
-# See AUTHORS.txt
-# SPDX-License-Identifier: MPL-2.0
-# This file is part of BERTrend.
-#
-
-# Starts the Topic Analysis application
-CUDA_VISIBLE_DEVICES=0 streamlit run --theme.primaryColor royalblue topic_analysis/app.py

bertrend-0.3.5/bertrend/demos/start_weak_signals_app.sh

@@ -1,9 +0,0 @@
-#
-# Copyright (c) 2024, RTE (https://www.rte-france.com)
-# See AUTHORS.txt
-# SPDX-License-Identifier: MPL-2.0
-# This file is part of BERTrend.
-#
-
-# Starts the Trend Analysis application
-CUDA_VISIBLE_DEVICES=0 streamlit run --theme.primaryColor royalblue weak_signals/app.py

bertrend-0.3.5/bertrend_apps/prospective_demo/.streamlit/config.toml

@@ -1,3 +0,0 @@
-[theme]
-primaryColor = "royalblue"
-baseFontSize = 13

bertrend-0.3.5/bertrend_apps/prospective_demo/start_demo.sh

@@ -1,9 +0,0 @@
-#
-# Copyright (c) 2024, RTE (https://www.rte-france.com)
-# See AUTHORS.txt
-# SPDX-License-Identifier: MPL-2.0
-# This file is part of BERTrend.
-#
-
-# Starts the Trend Analysis application
-CUDA_VISIBLE_DEVICES=0 streamlit run app.py