bertrend 0.4.6__tar.gz → 0.4.8__tar.gz

{bertrend-0.4.6 → bertrend-0.4.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bertrend
-Version: 0.4.6
+Version: 0.4.8
 Summary: BERTrend analyses topic evolution over time using state-of-the-art transformer models: topic modelling and LLM
 License: LICENSE.md
 License-File: AUTHORS.txt
@@ -12,6 +12,7 @@ Maintainer: Jérôme Picault
 Maintainer-email: jerome.picault@rte-france.com
 Requires-Python: >=3.12,<4.0
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Development Status :: 4 - Beta
 Requires-Dist: APScheduler (>=3.11.2)
 Requires-Dist: PyJWT (>=2.11.0)
 Requires-Dist: SQLAlchemy (>=2.0.46)
@@ -91,6 +92,7 @@ Description-Content-Type: text/markdown
 [![PyPi_Version](https://img.shields.io/pypi/v/bertrend.svg)](https://pypi.org/project/bertrend/)
 ![Static Badge](https://img.shields.io/badge/python-%3E%3D3.12-blue)
 ![GitHub License](https://img.shields.io/github/license/rte-france/BERTrend)
+[![Lines of Code](https://img.shields.io/endpoint?url=https%3A%2F%2Ftokei.kojix2.net%2Fbadge%2Fgithub%2Frte-france%2FBERTrend%2Flines)](https://tokei.kojix2.net/github/rte-france/BERTrend)
 
 ## News
 
@@ -122,7 +124,7 @@ The BERTrend repository provides a set of metrics, methods, and visualization to
 topic modeling in large text corpora. The tools are designed to help users detect, analyze, and visualize emerging
 topics and trends over time.
 
-The code base is illustrated via two main demos:
+The code base is illustrated via three main demos:
 
 ### Dynamic Topic Modeling Analysis
 
@@ -168,6 +170,24 @@ https://github.com/user-attachments/assets/d79368d9-d4e0-4324-8a98-a888f0ab3b65
 - Interpretation of these signals using LLMs
 - Inspection of the topic merging process using Sankey diagram
 
+### Prospective Analysis
+
+- [bertrend/bertrend_apps/prospective_demo](bertrend/bertrend_apps/prospective_demo): A comprehensive "Prospective
+  Analysis Demo" application for monitoring, analysis, and trend detection in real time (works with live data).
+
+#### Prospective Analysis Demonstrator Video
+
+(Coming soon)
+
+#### Demonstrator Features
+
+- Data flow configuration and collection status monitoring
+- Model status and monitoring by information source
+- Real-time signal analysis and trend exploration
+- Detailed topic analysis and interpretation
+- Automated report and newsletter generation
+- Comparative analysis between different periods
+
 ## Installation
 
 ### Environment Setup (.env and .venv)
@@ -428,6 +448,11 @@ Provides various statistics about topics:
 
 ![llm_interpretation](docs/images/llm_interpretation.png)
 
+## Article Scoring Service
+
+For service setup and API details, see:
+[`docs/services/article_scoring_service.md`](docs/services/article_scoring_service.md).
+
 ## Support
 
 Please use the 'Issues' section to report and discuss any problems with BERTrend.

{bertrend-0.4.6 → bertrend-0.4.8}/README.md

@@ -8,6 +8,7 @@
 [![PyPi_Version](https://img.shields.io/pypi/v/bertrend.svg)](https://pypi.org/project/bertrend/)
 ![Static Badge](https://img.shields.io/badge/python-%3E%3D3.12-blue)
 ![GitHub License](https://img.shields.io/github/license/rte-france/BERTrend)
+[![Lines of Code](https://img.shields.io/endpoint?url=https%3A%2F%2Ftokei.kojix2.net%2Fbadge%2Fgithub%2Frte-france%2FBERTrend%2Flines)](https://tokei.kojix2.net/github/rte-france/BERTrend)
 
 ## News
 
@@ -39,7 +40,7 @@ The BERTrend repository provides a set of metrics, methods, and visualization to
 topic modeling in large text corpora. The tools are designed to help users detect, analyze, and visualize emerging
 topics and trends over time.
 
-The code base is illustrated via two main demos:
+The code base is illustrated via three main demos:
 
 ### Dynamic Topic Modeling Analysis
 
@@ -85,6 +86,24 @@ https://github.com/user-attachments/assets/d79368d9-d4e0-4324-8a98-a888f0ab3b65
 - Interpretation of these signals using LLMs
 - Inspection of the topic merging process using Sankey diagram
 
+### Prospective Analysis
+
+- [bertrend/bertrend_apps/prospective_demo](bertrend/bertrend_apps/prospective_demo): A comprehensive "Prospective
+  Analysis Demo" application for monitoring, analysis, and trend detection in real time (works with live data).
+
+#### Prospective Analysis Demonstrator Video
+
+(Coming soon)
+
+#### Demonstrator Features
+
+- Data flow configuration and collection status monitoring
+- Model status and monitoring by information source
+- Real-time signal analysis and trend exploration
+- Detailed topic analysis and interpretation
+- Automated report and newsletter generation
+- Comparative analysis between different periods
+
 ## Installation
 
 ### Environment Setup (.env and .venv)
@@ -345,6 +364,11 @@ Provides various statistics about topics:
 
 ![llm_interpretation](docs/images/llm_interpretation.png)
 
+## Article Scoring Service
+
+For service setup and API details, see:
+[`docs/services/article_scoring_service.md`](docs/services/article_scoring_service.md).
+
 ## Support
 
 Please use the 'Issues' section to report and discuss any problems with BERTrend.

{bertrend-0.4.6 → bertrend-0.4.8}/bertrend/BERTrend.py

@@ -364,7 +364,7 @@ class BERTrend:
         # Check if both timestamps exist in topic models
         if new_model is None or self.last_topic_model is None:
             raise ValueError(
-                f"At least one topic model is not valid. You must fit the BERTrend model before merging models."
+                "At least one topic model is not valid. You must fit the BERTrend model before merging models."
             )
 
         # Get default BERTrend config if argument is not provided

{bertrend-0.4.6 → bertrend-0.4.8}/bertrend/bertrend_apps/common/apscheduler_utils.py

@@ -7,6 +7,7 @@ from __future__ import annotations
 
 import hashlib
 import os
+import re
 from contextlib import contextmanager
 from datetime import datetime
 from pathlib import Path
@@ -293,11 +294,15 @@ class APSchedulerUtils(SchedulerUtils):
 
     def remove_scrapping_for_user(self, feed_id: str, user: str | None = None):
         """Removes from the scheduler service the job matching the provided feed_id"""
+        # NB. user and model_id are escaped in case they contain special characters that can be interpreted in the regex search
         try:
             job_ids = self.find_jobs(
                 patterns={
                     "url": ".*/scrape-feed.*",
-                    "json_data": {"user": user, "model_id": feed_id},
+                    "json_data": {
+                        "user": re.escape(user),
+                        "model_id": re.escape(feed_id),
+                    },
                 }
             )
             self.remove_jobs(job_ids)
@@ -308,11 +313,15 @@ class APSchedulerUtils(SchedulerUtils):
 
     def remove_scheduled_training_for_user(self, model_id: str, user: str):
         """Removes from the crontab the training job matching the provided model_id"""
+        # NB. user and model_id are escaped in case they contain special characters that can be interpreted in the regex search
         try:
             job_ids = self.find_jobs(
                 patterns={
                     "url": ".*/train-new-model.*",
-                    "json_data": {"user": user, "model_id": model_id},
+                    "json_data": {
+                        "user": re.escape(user),
+                        "model_id": re.escape(model_id),
+                    },
                 }
             )
             self.remove_jobs(job_ids)
@@ -325,11 +334,15 @@ class APSchedulerUtils(SchedulerUtils):
         self, model_id: str, user: str
     ) -> bool:
         """Removes from the crontab the report generation job matching the provided model_id"""
+        # NB. user and model_id are escaped in case they contain special characters that can be interpreted in the regex search
         try:
             job_ids = self.find_jobs(
                 patterns={
                     "url": ".*/generate-report.*",
-                    "json_data": {"user": user, "model_id": model_id},
+                    "json_data": {
+                        "user": re.escape(user),
+                        "model_id": re.escape(model_id),
+                    },
                 }
             )
             self.remove_jobs(job_ids)
@@ -342,11 +355,15 @@ class APSchedulerUtils(SchedulerUtils):
         self, feed_id: str, user: str | None = None
     ) -> bool:
         """Checks if a given scrapping feed is active (registered with the service)."""
+        # NB. user and model_id are escaped in case they contain special characters that can be interpreted in the regex search
         try:
             job_ids = self.find_jobs(
                 patterns={
                     "url": ".*/scrape-feed.*",
-                    "json_data": {"user": user, "model_id": feed_id},
+                    "json_data": {
+                        "user": re.escape(user),
+                        "model_id": re.escape(feed_id),
+                    },
                 }
             )
             return len(job_ids) > 0
@@ -358,11 +375,15 @@ class APSchedulerUtils(SchedulerUtils):
 
     def check_if_learning_active_for_user(self, model_id: str, user: str) -> bool:
         """Checks if a given scrapping feed is active (registered in the crontab"""
+        # NB. user and model_id are escaped in case they contain special characters that can be interpreted in the regex search
         try:
             job_ids = self.find_jobs(
                 patterns={
                     "url": ".*/train-new-model.*",
-                    "json_data": {"user": user, "model_id": model_id},
+                    "json_data": {
+                        "user": re.escape(user),
+                        "model_id": re.escape(model_id),
+                    },
                 }
             )
             return len(job_ids) > 0
@@ -376,11 +397,15 @@ class APSchedulerUtils(SchedulerUtils):
         self, model_id: str, user: str
     ) -> bool:
         """Checks if automated report generation is active (registered in the crontab)"""
+        # NB. user and model_id are escaped in case they contain special characters that can be interpreted in the regex search
         try:
             job_ids = self.find_jobs(
                 patterns={
                     "url": ".*/generate-report.*",
-                    "json_data": {"user": user, "model_id": model_id},
+                    "json_data": {
+                        "user": re.escape(user),
+                        "model_id": re.escape(model_id),
+                    },
                 }
             )
             return len(job_ids) > 0
@@ -394,11 +419,15 @@ class APSchedulerUtils(SchedulerUtils):
         self, feed_id: str, user: str | None = None
     ) -> datetime | None:
         """Return the next scrapping date for the given feed_id and user"""
+        # NB. user and model_id are escaped in case they contain special characters that can be interpreted in the regex search
         try:
             jobs = self.find_jobs_description(
                 patterns={
                     "url": ".*/scrape-feed.*",
-                    "json_data": {"user": user, "model_id": feed_id},
+                    "json_data": {
+                        "user": re.escape(user),
+                        "model_id": re.escape(feed_id),
+                    },
                 }
             )
             if len(jobs) == 0:
@@ -414,11 +443,15 @@ class APSchedulerUtils(SchedulerUtils):
         self, model_id: str, user: str | None = None
     ) -> datetime | None:
         """Return the next scrapping date for the given feed_id and user"""
+        # NB. user and model_id are escaped in case they contain special characters that can be interpreted in the regex search
         try:
             jobs = self.find_jobs_description(
                 patterns={
                     "url": ".*/train-new-model.*",
-                    "json_data": {"user": user, "model_id": model_id},
+                    "json_data": {
+                        "user": re.escape(user),
+                        "model_id": re.escape(model_id),
+                    },
                 }
             )
             if len(jobs) == 0:

{bertrend-0.4.6 → bertrend-0.4.8}/bertrend/bertrend_apps/data_provider/data_provider.py

@@ -2,7 +2,6 @@
 #  See AUTHORS.txt
 #  SPDX-License-Identifier: MPL-2.0
 #  This file is part of BERTrend.
-import asyncio
 import os
 from abc import ABC, abstractmethod
 from pathlib import Path
@@ -10,6 +9,7 @@ from pathlib import Path
 import jsonlines
 import langdetect
 import pandas as pd
+import requests
 from dateutil import parser
 from goose3 import Goose
 from joblib import Parallel, delayed
@@ -17,12 +17,11 @@ from loguru import logger
 from newspaper import Article
 
 from bertrend.article_scoring.article_scoring import QualityLevel
-from bertrend.article_scoring.scoring_agent import score_articles
-from bertrend.utils.data_loading import TEXT_COLUMN
 from bertrend.bertrend_apps.data_provider.utils import (
     decode_google_news_url,
     wait_if_seen_url,
 )
+from bertrend.utils.data_loading import TEXT_COLUMN
 
 # Ensures to write with +rw for both user and groups
 os.umask(0o002)
@@ -85,27 +84,53 @@ class DataProvider(ABC):
 
     @staticmethod
     def evaluate_quality(
-        articles: list[dict], minimum_quality_level: QualityLevel
+        articles: list[dict], minimum_quality_level: QualityLevel = QualityLevel.AVERAGE
     ) -> list[dict]:
         texts = [article[TEXT_COLUMN] for article in articles]
+        article_scoring_service_url = os.getenv("ARTICLE_SCORING_SERVICE_URL")
+
+        if not article_scoring_service_url:
+            logger.warning(
+                "ARTICLE_SCORING_SERVICE_URL is not set; skipping article quality filtering."
+            )
+            return articles
+
+        try:
+            response = requests.post(
+                f"{article_scoring_service_url.rstrip('/')}/score",
+                json={"articles": texts},
+                timeout=300,
+            )
+            response.raise_for_status()
+            payload = response.json()
+            results = payload.get("results", [])
+        except requests.RequestException as exc:
+            logger.warning(
+                f"Article scoring service unavailable at {article_scoring_service_url}: {exc}. "
+                "Skipping article quality filtering."
+            )
+            return articles
 
-        # Evaluate quality of articles using a LLM agent
-        results = asyncio.run(score_articles(texts))
+        if len(results) != len(texts):
+            logger.warning(
+                "Article scoring service returned an unexpected number of results; "
+                "skipping article quality filtering."
+            )
+            return articles
 
-        # Filter out articles with quality < minimum_quality
+        # Filter out articles with quality < minimum_quality_level
         filtered_articles = [
             {
                 **article,
-                "quality_metrics": results[i].output.model_dump(),
-                "overall_quality": results[i].output.quality_level.name,
+                "quality_metrics": results[i]["quality_metrics"],
+                "overall_quality": results[i]["overall_quality"],
             }
-            # 1. Ensure output is NOT None before proceeding
             for i, article in enumerate(articles)
-            if results[i].output is not None
-            # 2. Then, check the quality level
-            and results[i].output.quality_level >= minimum_quality_level
+            if results[i].get("overall_quality") in QualityLevel.__members__
+            and QualityLevel[results[i]["overall_quality"]] >= minimum_quality_level
+            and not results[i].get("error")
         ]
-        assert len(results) == len(texts)
+
         logger.info(
             f"Filtered out {len(articles) - len(filtered_articles)} articles with quality < {minimum_quality_level.name}"
         )

{bertrend-0.4.6 → bertrend-0.4.8}/bertrend/bertrend_apps/exploration/curebot/tabs/tab1.py

@@ -31,7 +31,7 @@ def show() -> None:
         except Exception as e:
             logger.error(e)
             st.error(
-                f"Erreur lors du chargement des données. Vérifiez que vos données respectent le format Curebot attendu."
+                "Erreur lors du chargement des données. Vérifiez que vos données respectent le format Curebot attendu."
             )
 
     # If data is loaded

{bertrend-0.4.6 → bertrend-0.4.8}/bertrend/bertrend_apps/exploration/curebot/tabs/tab3.py

@@ -45,7 +45,7 @@ def show() -> None:
             st.button(
                 "Éditer",
                 on_click=edit_newsletter,
-                disabled=not "newsletter_text" in st.session_state,
+                disabled="newsletter_text" not in st.session_state,
             )
 
         # Show newsletter download button
@@ -65,7 +65,7 @@ def show() -> None:
                 file_name="newsletters.html",
                 mime="text/html",
                 data=data,
-                disabled=not "newsletter_text" in st.session_state,
+                disabled="newsletter_text" not in st.session_state,
             )
 
         # Show newsletter

{bertrend-0.4.6 → bertrend-0.4.8}/bertrend/bertrend_apps/prospective_demo/authentication.py

@@ -3,14 +3,17 @@
 #  SPDX-License-Identifier: MPL-2.0
 #  This file is part of BERTrend.
 import hmac
+import os
 
 import streamlit as st
 
-from bertrend.demos.demos_utils.icons import UNHAPPY_ICON
+from bertrend.bertrend_apps.prospective_demo.i18n import translate
+from bertrend.demos.demos_utils.icons import EMAIL_ICON, UNHAPPY_ICON
 
 
 def login_form():
     """Form with widgets to collect user information"""
+    st.header("")
     with st.form("Credentials"):
         st.text_input("Username", key="username")
         st.text_input("Password", type="password", key="password")
@@ -38,6 +41,16 @@ def logout():
         del st.session_state["username"]
 
 
+def show_contact_info():
+    contact = os.getenv("BERTREND_CONTACT_EMAIL", "contact@bertrend.com")
+    st.link_button(translate("contact_msg"), f"mailto:{contact}", icon=EMAIL_ICON)
+
+
+def show_project_info():
+    st.write("")
+    st.markdown(translate("project_info_msg"))
+
+
 def check_password() -> str | None:
     """Returns the user name if the user had a correct password, otherwise None."""
 
@@ -49,4 +62,8 @@ def check_password() -> str | None:
     login_form()
     if "password_correct" in st.session_state:
         st.error(f"{UNHAPPY_ICON} User not known or password incorrect")
+
+    show_contact_info()
+
+    show_project_info()
     return None

{bertrend-0.4.6 → bertrend-0.4.8}/bertrend/bertrend_apps/prospective_demo/feeds_config.py

@@ -20,6 +20,7 @@ from bertrend.bertrend_apps.prospective_demo.feeds_common import (
     read_user_feeds,
 )
 from bertrend.bertrend_apps.prospective_demo.i18n import translate
+from bertrend.bertrend_apps.prospective_demo.models_info import delete_model_config
 from bertrend.config.parameters import LANGUAGES
 from bertrend.demos.demos_utils.icons import (
     ADD_ICON,
@@ -337,7 +338,6 @@ def handle_delete(row_dict: dict):
                 feed_id=feed_id, user=st.session_state.username
             )
             delete_feed_config(feed_id)
-            logger.info(f"Flux {feed_id} supprimé !")
             # Remove from crontab associated training
             SCHEDULER_UTILS.remove_scheduled_training_for_user(
                 model_id=feed_id, user=st.session_state.username
@@ -349,6 +349,8 @@ def handle_delete(row_dict: dict):
                 SCHEDULER_UTILS.remove_scheduled_report_generation_for_user(
                     model_id=feed_id, user=st.session_state.username
                 )
+            delete_model_config(feed_id)
+            logger.info(f"Flux {feed_id} supprimé !")
             time.sleep(0.2)
             st.rerun()
     with col2:

{bertrend-0.4.6 → bertrend-0.4.8}/bertrend/bertrend_apps/prospective_demo/i18n_translations.py

@@ -3,10 +3,18 @@
 #  SPDX-License-Identifier: MPL-2.0
 #  This file is part of BERTrend.
 TRANSLATIONS = {
+    "project_info_msg": {
+        "fr": "BERTrend est un projet open-source développé par RTE. Visitez notre github pour plus d'informations: https://github.com/rte-france/BERTrend",
+        "en": "BERTrend is an open-source project developed by RTE. Visit our github for more information: https://github.com/rte-france/BERTrend",
+    },
+    "contact_msg": {
+        "en": "Questions, new account or bug reports? Contact us !",
+        "fr": "Des questions, un nouveau compte ou des rapports de bugs ? Contactez-nous !",
+    },
     # App.py translations
     "app_title": {
-        "fr": "BERTrend - Démo Veille & Analyse",
-        "en": "BERTrend - Monitoring & Analysis Demo",
+        "fr": "BERTrend - IA pour la Veille (Analyses & Tendances)",
+        "en": "BERTrend - AI for Monitoring, Analysis & Trends",
     },
     "check_data_button_label": {"fr": "Vérifier les données", "en": "Check data"},
     "tab_monitoring": {"fr": "Veilles", "en": "Monitoring"},

{bertrend-0.4.6 → bertrend-0.4.8}/bertrend/bertrend_apps/prospective_demo/models_info.py

@@ -13,6 +13,16 @@ import toml
 from loguru import logger
 
 from bertrend import load_toml_config
+from bertrend.bertrend_apps import SCHEDULER_UTILS
+from bertrend.bertrend_apps.prospective_demo import (
+    DEFAULT_ANALYSIS_CFG,
+    get_model_cfg_path,
+    get_user_models_path,
+)
+from bertrend.bertrend_apps.prospective_demo.i18n import translate
+from bertrend.bertrend_apps.prospective_demo.perf_utils import get_least_used_gpu
+from bertrend.bertrend_apps.prospective_demo.process_new_data import regenerate_models
+from bertrend.bertrend_apps.prospective_demo.utils import is_valid_email
 from bertrend.config.parameters import LANGUAGES
 from bertrend.demos.demos_utils.icons import (
     DELETE_ICON,
@@ -28,16 +38,6 @@ from bertrend.demos.streamlit_components.clickable_df_component import clickable
 from bertrend.demos.streamlit_components.input_with_pills_component import (
     input_with_pills,
 )
-from bertrend.bertrend_apps import SCHEDULER_UTILS
-from bertrend.bertrend_apps.prospective_demo import (
-    DEFAULT_ANALYSIS_CFG,
-    get_model_cfg_path,
-    get_user_models_path,
-)
-from bertrend.bertrend_apps.prospective_demo.i18n import translate
-from bertrend.bertrend_apps.prospective_demo.perf_utils import get_least_used_gpu
-from bertrend.bertrend_apps.prospective_demo.process_new_data import regenerate_models
-from bertrend.bertrend_apps.prospective_demo.utils import is_valid_email
 
 
 @st.cache_data(ttl=60)
@@ -302,6 +302,17 @@ def save_model_config(model_id: str, config: dict):
     logger.debug(f"Saved model analysis config {config} to {model_cfg_path}")
 
 
+def delete_model_config(model_id: str):
+    model_cfg_path = get_model_cfg_path(
+        user_name=st.session_state.username, model_id=model_id
+    )
+    if os.path.exists(model_cfg_path):
+        os.remove(model_cfg_path)
+        logger.info(f"Model config {model_id} removed")
+    else:
+        logger.warning(f"Model config {model_id} not found")
+
+
 @st.dialog(translate("dialog_confirmation"))
 def handle_delete_models(row_dict: dict):
     """Function to handle remove models from cache"""