bertrend 0.3.6__tar.gz → 0.3.8__tar.gz

{bertrend-0.3.6 → bertrend-0.3.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: bertrend
-Version: 0.3.6
+Version: 0.3.8
 Summary: BERTrend analyses topic evolution over time using state-of-the-art transformer models: topic modelling and LLM
 License: LICENSE.md
 Keywords: topics,BERTopic,transformers,weak signals,LLM
@@ -57,7 +57,7 @@ Requires-Dist: seaborn (>=0.13.2)
 Requires-Dist: sentence-transformers (==3.0.1)
 Requires-Dist: sentencepiece (>=0.2.0)
 Requires-Dist: starlette (>=0.46.2)
-Requires-Dist: streamlit (>=1.45.1)
+Requires-Dist: streamlit (>=1.46)
 Requires-Dist: streamlit-code-editor (>=0.1.22)
 Requires-Dist: thefuzz (>=0.22.1)
 Requires-Dist: tiktoken (>=0.8.0)

{bertrend-0.3.6 → bertrend-0.3.8}/bertrend/BERTopicModel.py

@@ -237,6 +237,7 @@ class BERTopicModel:
             Precomputed document embeddings.
         embedding_model : SentenceTransformer or str or None, optional
             Sentence transformer (or associated model name) model for embeddings.
+            NB. Important: if not set, the keywords describing the topics may not be correct (e.g. accents strip); therefore, it is recommended to set this parameter.
         zeroshot_topic_list : list[str] or None, optional
             List of topics used for zeroshot classification.
         zeroshot_min_similarity : float or None, optional

{bertrend-0.3.6 → bertrend-0.3.8}/bertrend/config/bertopic_default_config.toml

@@ -45,4 +45,4 @@ diversity = 0.3
 
 # Reduce outliers: https://maartengr.github.io/BERTopic/api/bertopic.html#bertopic._bertopic.BERTopic.reduce_outliers
 [reduce_outliers]
-strategy = "c-tf-idf"
+strategy = "c-tf-idf"

{bertrend-0.3.6 → bertrend-0.3.8}/bertrend/demos/demos_utils/data_loading_component.py

@@ -37,6 +37,9 @@ from bertrend.utils.data_loading import (
     split_data,
     TIMESTAMP_COLUMN,
     DataLoadingError,
+    _file_to_pd,
+    _clean_data,
+    _check_data,
 )
 
 FORMAT_ICONS = {
@@ -50,7 +53,31 @@ FORMAT_ICONS = {
 }
 
 
-@st.cache_data(ttl=60 * 60 * 24)
+@st.dialog(translate("column_selection"))
+def _select_alternative_columns(df: pd.DataFrame, message: str = ""):
+    st.warning(message, icon=WARNING_ICON),
+    if TEXT_COLUMN not in df.columns:
+        text_column = st.selectbox(
+            label=translate("text_column_selection"), options=df.columns
+        )
+    else:
+        text_column = None
+    if TIMESTAMP_COLUMN not in df.columns:
+        timestamp_column = st.selectbox(
+            label=translate("timestamp_column_selection"), options=df.columns
+        )
+    else:
+        timestamp_column = None
+    if st.button("Submit"):
+        if text_column:
+            df[TEXT_COLUMN] = df[text_column]
+        if timestamp_column:
+            df[TIMESTAMP_COLUMN] = df[timestamp_column]
+        st.session_state["modified_df"] = df
+        st.rerun()
+
+
+#    @st.cache_data(ttl=60 * 60 * 24)
 def _process_uploaded_files(
     files: list[UploadedFile],
 ) -> list[pd.DataFrame]:
@@ -61,20 +88,23 @@ def _process_uploaded_files(
             with open(tmpdir + "/" + f.name, "wb") as tmp_file:
                 tmp_file.write(f.getvalue())
             if tmp_file is not None:
+                if "modified_df" in st.session_state:
+                    df = st.session_state["modified_df"]
+                    st.session_state.pop("modified_df")
+                else:
+                    df = _file_to_pd(Path(tmp_file.name))
                 try:
-                    df = load_data(
-                        Path(tmp_file.name),
-                        SessionStateManager.get("language", "French"),
-                    )
+                    _check_data(df, Path(tmp_file.name))
                 except DataLoadingError as dle:
-                    st.warning(
-                        translate("error_loading_file").format(
-                            file_name=f.name, error=dle
-                        ),
-                        icon=WARNING_ICON,
+                    message = translate("error_loading_file").format(
+                        file_name=f.name, error=dle
                     )
+                    if "modified_df" in st.session_state:
+                        st.session_state.pop("modified_df")
+                    _select_alternative_columns(df, message)
                     df = None
                 if df is not None:
+                    df = _clean_data(df, SessionStateManager.get("language", "French"))
                     dataframes.append(df)
         return dataframes
 
@@ -99,7 +129,12 @@ def _load_files(
                 ),
                 icon=WARNING_ICON,
             )
-            df = None
+            _select_alternative_columns(df)
+            df = (
+                st.session_state["modified_df"]
+                if "modified_df" in st.session_state
+                else None
+            )
         if df is not None:
             dfs.append(df)
     return dfs

{bertrend-0.3.6 → bertrend-0.3.8}/bertrend/demos/demos_utils/embed_documents_component.py

@@ -31,6 +31,9 @@ def display_embed_documents_component():
                     local=False,
                     url=SessionStateManager.get("embedding_service_url"),
                 )
+                SessionStateManager.set(
+                    "embedding_model_name", embedding_service.embedding_model_name
+                )
 
             texts = SessionStateManager.get_dataframe("time_filtered_df")[
                 TEXT_COLUMN

{bertrend-0.3.6 → bertrend-0.3.8}/bertrend/demos/demos_utils/i18n_translations.py

@@ -137,4 +137,16 @@ TRANSLATIONS = {
         "fr": "Paramètres et contrôles",
         "en": "Settings and Controls",
     },
+    "column_selection": {
+        "fr": "Sélection des colonnes",
+        "en": "Column Selection",
+    },
+    "text_column_selection": {
+        "fr": "Sélection de la colonne contenant le texte",
+        "en": "Select Column Containing Text",
+    },
+    "timestamp_column_selection": {
+        "fr": "Sélection de la colonne contenant l'horodatage",
+        "en": "Select Column Containing Timestamp",
+    },
 }

{bertrend-0.3.6 → bertrend-0.3.8}/bertrend/demos/topic_analysis/demo_pages/explore_topics.py

@@ -282,27 +282,23 @@ def create_topic_documents(
     return folder_name, documents
 
 
+@st.cache_data
 def _display_topic_description(filtered_df: pd.DataFrame):
     """Display a human-readable description of the selected topic using a LLM."""
-    if st.button(
-        translate("generate_topic_description"),
-        type="primary",
-        use_container_width=True,
-    ):
-        with st.spinner(translate("generating_description")):
-            language_code = (
-                "fr"
-                if SessionStateManager.get("internationalization_language") == "French"
-                else "en"
-            )
-            gpt_description = generate_topic_description(
-                st.session_state["topic_model"],
-                st.session_state["selected_topic_number"],
-                filtered_df,
-                language_code=language_code,
-            )
-        with st.container(border=True):
-            st.markdown(f"### {gpt_description.title}\n{gpt_description.description}")
+    with st.spinner(translate("generating_description")):
+        language_code = (
+            "fr"
+            if SessionStateManager.get("internationalization_language") == "fr"
+            else "en"
+        )
+        gpt_description = generate_topic_description(
+            st.session_state["topic_model"],
+            st.session_state["selected_topic_number"],
+            filtered_df,
+            language_code=language_code,
+        )
+    with st.container(border=True):
+        st.markdown(f"### {gpt_description.title}\n{gpt_description.description}")
 
 
 def main():

{bertrend-0.3.6 → bertrend-0.3.8}/bertrend/demos/topic_analysis/demo_pages/newsletters_generation.py

@@ -6,6 +6,7 @@ import inspect
 
 import pandas as pd
 import streamlit as st
+import streamlit.components.v1 as components
 from pathlib import Path
 
 from bertrend.demos.demos_utils.i18n import translate
@@ -16,13 +17,17 @@ from bertrend.demos.demos_utils.state_utils import (
     register_multiple_widget,
     SessionStateManager,
 )
+from bertrend.llm_utils.newsletter_model import Newsletter
 from bertrend.services.summary.abstractive_summarizer import AbstractiveSummarizer
 from bertrend.services.summary.chatgpt_summarizer import GPTSummarizer
 from bertrend.services.summary.extractive_summarizer import (
     ExtractiveSummarizer,
     EnhancedExtractiveSummarizer,
 )
-from bertrend.llm_utils.newsletter_features import generate_newsletter, md2html
+from bertrend.llm_utils.newsletter_features import (
+    generate_newsletter,
+    render_newsletter_html,
+)
 
 
 # Define summarizer options
@@ -34,9 +39,7 @@ SUMMARIZER_OPTIONS_MAPPER = {
 }
 
 
-def generate_newsletter_wrapper(
-    df: pd.DataFrame, df_split: pd.DataFrame
-) -> tuple[str, str, str]:
+def generate_newsletter_wrapper(df: pd.DataFrame, df_split: pd.DataFrame) -> Newsletter:
     """Wrapper function to generate newsletter based on user settings."""
     top_n_topics = (
         None
@@ -51,7 +54,7 @@ def generate_newsletter_wrapper(
 
     language_code = (
         "fr"
-        if SessionStateManager.get("internationalization_language") == "French"
+        if SessionStateManager.get("internationalization_language") == "fr"
         else "en"
     )
 
@@ -172,10 +175,14 @@ def main():
 
     # Display generated newsletters
     if "newsletters" in st.session_state:
-        st.components.v1.html(
-            md2html(
-                st.session_state["newsletters"][0],
-                Path(inspect.getfile(generate_newsletter)).parent / "newsletter.css",
+        components.html(
+            render_newsletter_html(
+                st.session_state["newsletters"],
+                html_template=Path(inspect.getfile(generate_newsletter)).parent
+                / "newsletter_template.html",
+                custom_css=Path(inspect.getfile(generate_newsletter)).parent
+                / "newsletter.css",
+                language=SessionStateManager.get("internationalization_language"),
             ),
             height=800,
             scrolling=True,

{bertrend-0.3.6 → bertrend-0.3.8}/bertrend/demos/topic_analysis/demo_pages/training_page.py

@@ -127,6 +127,7 @@ def train_model():
         topic_model_output = topic_model.fit(
             docs=dataset,
             embeddings=embeddings,
+            embedding_model=SessionStateManager.get("embedding_model_name"),
         )
         bertopic = topic_model_output.topic_model
 

bertrend-0.3.8/bertrend/llm_utils/newsletter.css

@@ -0,0 +1,145 @@
+/*
+ * Copyright (c) 2024, RTE (https://www.rte-france.com)
+ * See AUTHORS.txt
+ * SPDX-License-Identifier: MPL-2.0
+ * This file is part of BERTrend.
+ */
+
+/* General Body Styles */
+body {
+    font-family: Arial, sans-serif;
+    margin: 0;
+    padding: 0;
+    background-color: #f4f4f9;
+    color: #333;
+}
+
+.container {
+    max-width: 800px;
+    margin: 0 auto;
+    padding: 20px;
+    background-color: #fff;
+}
+
+h1 {
+    font-size: 32px;
+    color: #333;
+}
+
+.period {
+    font-size: 18px;
+    color: #555;
+}
+
+/* Topic Styles */
+.topic {
+    margin-top: 30px;
+    padding-left: 20px;  /* Added padding to make room for the colored bar */
+    border-left: 10px solid transparent;  /* Default to transparent */
+}
+
+/* Topic Types - Adjusted to apply colors to the entire topic */
+.topic.strong {
+    border-left-color: #4caf50; /* Green for strong */
+}
+
+.topic.weak {
+    border-left-color: #ff9800; /* Orange for weak */
+}
+
+.topic.noise {
+    border-left-color: #9e9e9e; /* Grey for noise */
+}
+
+h2 {
+    font-size: 24px;
+    color: #c50000;
+}
+
+.summary {
+    margin: 10px 0;
+    font-size: 16px;
+    color: #666;
+}
+
+/* Article Styles */
+.articles {
+    margin-top: 15px;
+}
+
+.article {
+    margin-bottom: 15px;
+    padding: 10px;
+    background-color: #f9f9f9;
+    border-left: none; /* Removed the vertical bar for articles */
+}
+
+.article h3 {
+    font-size: 18px;
+    color: #2a2a2a;
+}
+
+/* Styling for the Article Title Link */
+.article a {
+    text-decoration: none; /* Remove default underline */
+    color: #0066cc; /* Blue color for the link */
+    font-weight: bold; /* Make the link title bold */
+    transition: all 0.3s ease; /* Smooth transition for hover effect */
+}
+
+/* Hover Effect */
+.article a:hover {
+    color: #005bb5; /* Darker blue on hover */
+    text-decoration: underline; /* Underline on hover for better visual feedback */
+    border-bottom: 2px solid #005bb5; /* Optional: Add a border for emphasis */
+}
+
+/* Date and Source Inline */
+.timestamp {
+    display: flex;
+    align-items: center;
+    font-size: 14px;
+    color: grey;
+    margin-bottom: 10px;
+}
+
+.timestamp .separator {
+    margin: 0 5px;
+}
+
+.date {
+    font-weight: bold;
+}
+
+.source {
+    font-style: italic;
+}
+
+/* Hashtags */
+.hashtags {
+    margin-top: 7px;
+}
+.hashtags span {
+    display: inline-block;
+    margin-right: 8px;
+    padding: 2px 8px;
+    background: #e8eefa;
+    color: #205296;
+    border-radius: 6px;
+    font-size: 12px;
+    font-weight: 500;
+    letter-spacing: 0.2px;
+    margin-bottom: 2px;
+    transition: background 0.16s;
+}
+.hashtags span:hover {
+    background: #bfd6f6;
+}
+
+/* Footer Styles */
+.footer {
+    margin-top: 40px;
+    text-align: center;
+    font-size: 12px;
+    color: #aaa;
+}