bertrend 0.3.14__tar.gz → 0.3.16__tar.gz

{bertrend-0.3.14 → bertrend-0.3.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bertrend
-Version: 0.3.14
+Version: 0.3.16
 Summary: BERTrend analyses topic evolution over time using state-of-the-art transformer models: topic modelling and LLM
 License: LICENSE.md
 License-File: AUTHORS.txt

{bertrend-0.3.14 → bertrend-0.3.16}/bertrend/BERTrend.py

@@ -282,6 +282,9 @@ class BERTrend:
             (period, group) for period, group in grouped_data.items() if not group.empty
         ]
 
+        failed_periods = []
+        successful_periods = []
+
         for i, (period, group) in enumerate(non_empty_groups):
             try:
                 logger.info(f"Training topic model {i+1}/{len(non_empty_groups)}...")
@@ -308,15 +311,26 @@ class BERTrend:
                 self.last_topic_model_timestamp = period
 
                 logger.debug(f"Successfully processed period: {period}")
+                successful_periods.append(period)
 
             except Exception as e:
                 logger.error(f"Error processing period {period}: {e}")
                 logger.exception("Traceback:")
+                failed_periods.append(period)
                 continue  # TODO: better error handling
 
         self._is_fitted = True
 
-        logger.success("Finished training all topic models")
+        if failed_periods:
+            logger.error(
+                f"Training completed with failures for {len(failed_periods)} periods: {failed_periods}"
+            )
+            if successful_periods:
+                logger.warning(
+                    f"Successfully trained topic models for {len(successful_periods)} periods: {successful_periods}"
+                )
+        else:
+            logger.success("Finished training all topic models")
 
     def merge_models_with(
         self,

{bertrend-0.3.14 → bertrend-0.3.16}/bertrend_apps/prospective_demo/.streamlit/secrets.toml

@@ -11,3 +11,4 @@ celine = "celine"
 lucas = "lucas"
 laurent = "laurent"
 diese = "diese"
+test = "test"

{bertrend-0.3.14 → bertrend-0.3.16}/bertrend_apps/prospective_demo/__init__.py

@@ -33,7 +33,7 @@ DEFAULT_ANALYSIS_CFG = {
     "model_config": {
         "granularity": DEFAULT_GRANULARITY,
         "window_size": DEFAULT_WINDOW_SIZE,
-        "language": "French",
+        "language": "English",
         "split_by_paragraph": True,
     },
     "analysis_config": {

{bertrend-0.3.14 → bertrend-0.3.16}/bertrend_apps/prospective_demo/i18n_translations.py

@@ -383,8 +383,8 @@ TRANSLATIONS = {
         "fr": "Retenir seulement les articles de meilleure qualité",
     },
     "evaluate_articles_quality_help": {
-        "en": "Score each article using a LLM-based approach ($$) and keep only high-quality articles",
-        "fr": "Evalue chaque article avec une approche basée sur des LLM ($$) et ne retient que les articles de meilleure qualité",
+        "en": "Score each article using a LLM-based approach (💶) and keep only high-quality articles",
+        "fr": "Evalue chaque article avec une approche basée sur des LLM (💶) et ne retient que les articles de meilleure qualité",
     },
     "minimum_quality_level": {
         "en": "Minimum quality level",

{bertrend-0.3.14 → bertrend-0.3.16}/bertrend_apps/prospective_demo/models_info.py

@@ -60,7 +60,17 @@ def models_monitoring():
         try:
             load_model_config(model_id)
         except Exception:
+            # create default config if not found
             st.session_state.model_analysis_cfg[model_id] = DEFAULT_ANALYSIS_CFG
+            # align language with the feed
+            st.session_state.model_analysis_cfg[model_id]["model_config"][
+                "language"
+            ] = (
+                "French"
+                if st.session_state.user_feeds[model_id]["data-feed"]["language"]
+                == "fr"
+                else "English"
+            )
             # special case for split by paragraphs
             st.session_state.model_analysis_cfg[model_id]["model_config"][
                 "split_by_paragraph"
@@ -68,6 +78,7 @@ def models_monitoring():
                 st.session_state.user_feeds[model_id]["data-feed"]["provider"]
                 != "arxiv"
             )
+            save_model_config(model_id, st.session_state.model_analysis_cfg[model_id])
 
         list_models = get_models_info(model_id)
         displayed_list.append(

{bertrend-0.3.14 → bertrend-0.3.16}/bertrend_apps/prospective_demo/process_new_data.py

@@ -4,6 +4,7 @@
 #  This file is part of BERTrend.
 from datetime import timedelta
 from pathlib import Path
+from concurrent.futures import ThreadPoolExecutor, as_completed
 
 import pandas as pd
 import typer
@@ -78,7 +79,7 @@ def get_relevant_model_config(
     window_size = model_analysis_cfg["model_config"]["window_size"]
     language = model_analysis_cfg["model_config"]["language"]
     if language not in ["French", "English"]:
-        language = "French"
+        language = "English"
     return granularity, window_size, language
 
 
@@ -89,6 +90,7 @@ def generate_llm_interpretation(
     df_name: str,
     output_path: Path,
     top_k: int = DEFAULT_TOP_K,
+    max_workers: int = 5,
 ):
     """
     Generate detailed analysis for the top k topics using parallel processing.
@@ -100,23 +102,53 @@ def generate_llm_interpretation(
         df_name: Name of the DataFrame for output
         output_path: Path to save the results
         top_k: Number of top topics to analyze
+        max_workers: Maximum number of concurrent workers for parallel processing
     """
 
-    interpretation = []
-    for topic in df.sort_values(by=["Latest_Popularity"], ascending=False).head(top_k)[
-        "Topic"
-    ]:
-        summary, analysis = analyze_signal(bertrend, topic, reference_timestamp)
-        if not summary or not analysis:
-            logger.warning(f"Skipping topic {topic} as analysis of signal failed.")
-            continue
-        interpretation.append(
-            {
+    # Get topics sorted by popularity
+    topics = (
+        df.sort_values(by=["Latest_Popularity"], ascending=False)
+        .head(top_k)["Topic"]
+        .tolist()
+    )
+
+    def process_topic(topic):
+        """Process a single topic and return the result or None if failed"""
+        try:
+            summary, analysis = analyze_signal(bertrend, topic, reference_timestamp)
+            if not summary or not analysis:
+                logger.warning(f"Skipping topic {topic} as analysis of signal failed.")
+                return None
+            return {
                 "topic": topic,
                 "summary": summary.model_dump_json(),
                 "analysis": analysis.model_dump_json(),
             }
-        )
+        except Exception as e:
+            logger.error(f"Error processing topic {topic}: {str(e)}")
+            return None
+
+    interpretation = []
+
+    # Use ThreadPoolExecutor for parallel processing of I/O-bound analyze_signal calls
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        # Submit all topic analysis tasks
+        future_to_topic = {
+            executor.submit(process_topic, topic): topic for topic in topics
+        }
+
+        # Collect results as they complete
+        for future in as_completed(future_to_topic):
+            topic = future_to_topic[future]
+            try:
+                result = future.result()
+                if result:
+                    interpretation.append(result)
+            except Exception as e:
+                logger.error(f"Exception occurred for topic {topic}: {str(e)}")
+
+    # Sort results by topic number to maintain consistent output order
+    interpretation.sort(key=lambda x: x["topic"])
 
     # Save interpretation
     output_file_name = output_path / f"{df_name}_interpretation.jsonl"

bertrend-0.3.16/bertrend_apps/tests/__init__.py

@@ -0,0 +1,4 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.