bertrend 0.3.1__py3-none-any.whl

bertrend_apps/exploration/curebot/old_veille_analyse.py

@@ -0,0 +1,298 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+import inspect
+import pandas as pd
+import streamlit as st
+from loguru import logger
+from streamlit.runtime.uploaded_file_manager import UploadedFile
+
+from bertrend.services.summary.chatgpt_summarizer import GPTSummarizer
+from bertrend.train import train_BERTopic
+from bertrend.utils.data_loading import (
+    load_data,
+    TIMESTAMP_COLUMN,
+    enhanced_split_df_by_paragraphs,
+    clean_dataset,
+)
+
+from bertrend_apps.data_provider.curebot_provider import CurebotProvider
+from bertrend.llm_utils.newsletter_features import generate_newsletter, md2html
+
+COLUMN_URL = "url"
+MIN_TEXT_LENGTH = 150
+EMBEDDING_MODEL_NAME = "dangvantuan/sentence-camembert-large"
+TOP_N_WORDS = 5
+# EMBEDDING_MODEL_NAME = "antoinelouis/biencoder-camembert-base-mmarcoFR"
+
+css_style = Path(inspect.getfile(generate_newsletter)).parent / "newsletter.css"
+
+if "topic_detection_disabled" not in st.session_state:
+    st.session_state.topic_detection_disabled = False
+if "newsletter_disabled" not in st.session_state:
+    st.session_state.newsletter_disabled = False
+if "import_expanded" not in st.session_state:
+    st.session_state.import_expanded = True
+if "st.session_state.topic_expanded" not in st.session_state:
+    st.session_state.topic_expanded = True
+
+
+@st.cache_data
+def parse_data_from_files(files: list[UploadedFile]) -> pd.DataFrame:
+    """Read a list of Excel files and return a single dataframe containing the data"""
+    dataframes = []
+
+    with TemporaryDirectory() as tmpdir:
+        for f in files:
+            with open(tmpdir + "/" + f.name, "wb") as tmp_file:
+                tmp_file.write(f.getvalue())
+                print(tmp_file.name)
+
+            if tmp_file is not None:
+                with st.spinner(f"Analyse des articles de: {f.name}"):
+                    provider = CurebotProvider(curebot_export_file=Path(tmp_file.name))
+                    articles = provider.get_articles()
+                    articles_path = Path(tmpdir) / (f.name + ".jsonl")
+                    provider.store_articles(articles, articles_path)
+                    df = (
+                        load_data(articles_path)
+                        .sort_values(by=TIMESTAMP_COLUMN, ascending=False)
+                        .reset_index(drop=True)
+                        .reset_index()
+                    )
+                    dataframes.append(df)
+
+        # Concat all dataframes
+        df_concat = pd.concat(dataframes, ignore_index=True)
+        df_concat = df_concat.drop_duplicates(subset=COLUMN_URL, keep="first")
+        return df_concat
+
+
+@st.cache_data
+def parse_data_from_feed(feed_url):
+    """Return a single dataframe containing the data obtained from the feed"""
+    with TemporaryDirectory() as tmpdir:
+        with st.spinner(f"Analyse des articles de: {feed_url}"):
+            provider = CurebotProvider(feed_url=feed_url)
+            articles = provider.get_articles()
+            articles_path = Path(tmpdir) / "feed.jsonl"
+            provider.store_articles(articles, articles_path)
+            return (
+                load_data(articles_path)
+                .sort_values(by=TIMESTAMP_COLUMN, ascending=False)
+                .reset_index(drop=True)
+                .reset_index()
+            )
+
+
+def split_data():
+    st.session_state["df_split"] = (
+        enhanced_split_df_by_paragraphs(st.session_state["df"])
+        .drop("index", axis=1)
+        .sort_values(
+            by=TIMESTAMP_COLUMN,
+            ascending=False,
+        )
+        .reset_index(drop=True)
+        .reset_index()
+    )
+
+    # Clean dataset using min_text_length
+    st.session_state["df_split"] = clean_dataset(
+        st.session_state["df_split"],
+        MIN_TEXT_LENGTH,
+    )
+
+
+def train_model():
+    (
+        st.session_state["topic_model"],
+        st.session_state["topics"],
+        _,
+        _,
+        _,
+        _,
+    ) = train_BERTopic(
+        full_dataset=st.session_state["df_split"],
+        indices=None,
+        embedding_model_name=EMBEDDING_MODEL_NAME,
+        use_cache=False,
+        top_n_words=TOP_N_WORDS,
+    )
+
+
+@st.experimental_dialog("Newsletter preview", width="large")
+def preview_newsletter():
+    content = md2html(st.session_state["final_newsletter"], css_style=css_style)
+    st.html(content)
+
+
+def import_data():
+    with st.expander(
+        "**Import des données de Curebot**", expanded=st.session_state.import_expanded
+    ):
+        # uploader
+        st.text_input(
+            "URL du flux ATOM / RSS",
+            value=(
+                st.session_state["curebot_rss_url"]
+                if "curebot_rss_url" in st.session_state
+                else ""
+            ),
+            key="curebot_rss_url",
+            help="Saisir le l'URL complète du flux Curebot à importer (par ex. https://api-a1.beta.curebot.io/v1/atom-feed/smartfolder/a5b14e159caa4cb5967f94e84640f602)",
+        )
+        uploaded_files = st.file_uploader(
+            "Fichiers Excel (format Curebot .xlsx)",
+            accept_multiple_files=True,
+            help="Glisser/déposer dans cette zone les exports Curebot au format Excel",
+        )
+
+    # check content
+    if uploaded_files or "curebot_rss_url" in st.session_state:
+        if uploaded_files:
+            st.session_state["df"] = parse_data_from_files(uploaded_files)
+        elif st.session_state["curebot_rss_url"]:
+            st.session_state["df"] = parse_data_from_feed(
+                st.session_state["curebot_rss_url"]
+            )
+
+        # split and clean data
+        if "df" in st.session_state:
+            split_data()
+            logger.info(f"Size of dataset: {len(st.session_state['df_split'])}")
+
+
+def display_data():
+    if "df" in st.session_state:
+        st.session_state.import_expanded = False
+        # Affichage du contenu du fichier Excel
+        with st.expander("**Contenu des données**", expanded=False):
+            st.dataframe(st.session_state["df"])
+
+
+def detect_topics():
+    if "df_split" in st.session_state:
+        st.session_state.import_expanded = False
+        with st.expander(
+            "**Détection de topics**", expanded=st.session_state.topic_expanded
+        ):
+            # Topic detection
+            col1, col2 = st.columns(2)
+            with col1:
+                st.button(
+                    "Détection des topics",
+                    on_click=train_model,
+                    key="topic_detection",
+                    type="primary",
+                    disabled=st.session_state.topic_detection_disabled,
+                )
+            with col2:
+                if "topic_model" in st.session_state:
+                    st.info(
+                        f"Nombre de topics: {len(st.session_state['topic_model'].get_topic_info()) - 1}"
+                    )
+
+
+def newsletter_creation():
+    # Newsletter creation
+    if "topic_model" in st.session_state.keys():
+        st.session_state.topic_expanded = False
+        with st.expander("**Création de la newsletters**", expanded=True):
+            # st.session_state.topic_detection_disabled = True
+            generation_button = st.button(
+                "Génération de newsletters",
+                on_click=create_newsletter,
+                type="primary",
+                disabled=st.session_state.newsletter_disabled,
+            )
+
+            # Edit manually newsletters
+            if "newsletters" in st.session_state.keys():
+                st.text_area(
+                    "Contenu éditable de la newsletters (faire CTRL+ENTREE pour prendre en compte les modifications)",
+                    value=(
+                        st.session_state["newsletters"]
+                        if (
+                            "final_newsletter" not in st.session_state
+                            or generation_button
+                        )
+                        else st.session_state["final_newsletter"]
+                    ),
+                    height=400,
+                    key="final_newsletter",
+                )
+
+                if "final_newsletter" in st.session_state:
+                    col1, col2 = st.columns(2)
+                    with col1:
+                        st.button("Preview", on_click=preview_newsletter)
+                    with col2:
+                        # Newsletter download
+                        st.download_button(
+                            "Téléchargement",
+                            file_name="newsletters.html",
+                            data=md2html(
+                                st.session_state["final_newsletter"],
+                                css_style=css_style,
+                            ),
+                            type="primary",
+                        )
+
+
+def main_page():
+    """Main page rendering"""
+    # title
+    st.title("Wattelse Veille & Analyse")
+    import_data()
+    display_data()
+    detect_topics()
+    newsletter_creation()
+
+
+def options():
+    with st.sidebar:
+        st.title("Réglages")
+
+        st.slider(
+            "Nombre max de topics",
+            min_value=1,
+            max_value=10,
+            value=5,
+            key="newsletter_nb_topics",
+        )
+
+        st.slider(
+            "Nombre max d'articles par topics",
+            min_value=1,
+            max_value=10,
+            value=5,
+            key="newsletter_nb_docs",
+        )
+
+        st.slider(
+            "Longueur des synthèses (# phrases)",
+            min_value=1,
+            max_value=10,
+            value=4,
+            key="nb_sentences",
+        )
+
+        st.selectbox(
+            "Moteur de résumé",
+            ("gpt-4o-mini", "gpt-4o", "gpt-3.5-turbo"),
+            key="openai_model_name",
+        )
+
+
+def main():
+    options()
+    main_page()
+
+
+# Main
+main()

bertrend_apps/exploration/curebot/prompts.py

@@ -0,0 +1,24 @@
+TOPIC_DESCRIPTION_SYSTEM_PROMPT = """
+Vous êtes expert en veille d'actualité et en anaylse thématique.
+Dans le contexte d'une analyse d'articles de presse, plusieurs articles ont été regroupés en un même thème.
+Votre tâche est de générer un titre pour ce thème sur la base des articles qui appartiennent à ce thème.
+A partir de la liste des articles fournie (titre et contenu), rédigez un titre pour le thème.
+Le titre doit être concis (maximum 5 mots) et représenter au mieux la spécificité du thème.
+Répondez sous la forme d'un JSON suivant le format ci-dessous :
+{
+    "titre": "<votre titre du thème>"
+}
+"""
+
+TOPIC_SUMMARY_SYSTEM_PROMPT = """
+Vous êtes expert en veille d'actualité et en anaylse thématique.
+Dans le contexte d'une analyse d'articles de presse, plusieurs articles ont été regroupés en un même thème.
+Votre tâche est de générer un résumé pour ce thème sur la base des articles qui appartiennent à ce thème.
+A partir de la liste des articles fournie (titre et contenu), rédigez un résumé pour le thème.
+Le résumé doit être concis (maximum 100 mots) et représenter au mieux la spécificité du thème.
+Il doit pas commencer par "Les articles parlent de..." ou équivalent et doit être écrit dans un style journalistique.
+Répondez sous la forme d'un JSON suivant le format ci-dessous :
+{
+    "résumé": "<votre résumé du thème>"
+}
+"""

bertrend_apps/exploration/curebot/start_newsletter_generator.sh

@@ -0,0 +1,8 @@
+#
+# Copyright (c) 2024, RTE (https://www.rte-france.com)
+# See AUTHORS.txt
+# SPDX-License-Identifier: MPL-2.0
+# This file is part of BERTrend.
+#
+
+OPENAI_API_KEY=$OPENAI_API_KEY_VEILLE CUDA_VISIBLE_DEVICES=0 streamlit run  --theme.primaryColor royalblue --server.port 8686 veille_analyse.py

bertrend_apps/exploration/curebot/tabs/tab1.py

@@ -0,0 +1,150 @@
+import pandas as pd
+from loguru import logger
+import streamlit as st
+
+from bertrend_apps.exploration.curebot.app_utils import (
+    TAGS_COLUMN,
+    TEXT_COLUMN,
+    TIMESTAMP_COLUMN,
+    TITLE_COLUMN,
+    TOP_N_WORDS,
+    concat_data_from_files,
+    fit_bertopic,
+    get_embeddings,
+    get_improved_topic_description,
+)
+
+
+def show() -> None:
+    # Data uploading component
+    upload_data()
+
+    # Load data into dataframe
+    if st.session_state.get("uploaded_files"):
+        try:
+            preprocess_data()
+        except Exception as e:
+            logger.error(e)
+            st.error(
+                f"Erreur lors du chargement des données. Vérifiez que vos données respectent le format Curebot attendu."
+            )
+
+    # If data is loaded
+    if "df" in st.session_state:
+        # Show data
+        with st.expander("Voir les données", expanded=False):
+            st.write(st.session_state["df"])
+
+        # Button to train model
+        if st.button("Détecter les sujets", type="primary"):
+            with st.spinner("Détection des sujets..."):
+                # Train model
+                train_model()
+
+        # If topic model is trained, update df with topics and llm description
+        if "topic_model" in st.session_state:
+            st.session_state["df"]["topics"] = st.session_state["topics"]
+            with st.spinner("Génération des titres des sujets..."):
+                st.session_state["topics_info"]["llm_description"] = (
+                    get_improved_topic_description(
+                        st.session_state["df"], st.session_state["topics_info"]
+                    )
+                )
+            st.success("Sujets détectés, voir l'onglet résultats.")
+
+
+def upload_data() -> None:
+    """
+    Data uploading component for Curebot format.
+    Sets in sessions_state:
+      - "uploaded_files": list of uploaded files
+    """
+    # Excel files input
+    st.session_state["uploaded_files"] = st.file_uploader(
+        "Fichiers Excel au format rapport Curebot `.xlsx`",
+        accept_multiple_files=True,
+        help="Glisser/déposer dans cette zone les exports Curebot au format Excel",
+    )
+
+
+def preprocess_data() -> None:
+    """
+    Preprocess data from uploaded files.
+    Sets in session_state:
+      - "df": dataframe with all data
+    """
+    # Concatenate uploaded Excel files into a single dataframe
+    df = concat_data_from_files(st.session_state["uploaded_files"])
+
+    # Remove duplicates based on title and text columns
+    df = df.drop_duplicates(subset=[TITLE_COLUMN, TEXT_COLUMN]).reset_index(drop=True)
+
+    # Remove rows where text is empty
+    df = df[df[TEXT_COLUMN].notna()].reset_index(drop=True)
+
+    # Sort df by date
+    df[TIMESTAMP_COLUMN] = pd.to_datetime(df[TIMESTAMP_COLUMN])
+    df = df.sort_values(by=TIMESTAMP_COLUMN, ascending=False).reset_index(drop=True)
+
+    st.session_state["df"] = df
+
+
+def train_model() -> None:
+    """
+    Train a BERTopic model based on provided data.
+    Sets in session_state:
+      - "embeddings": embeddings of the dataset
+      - "topic_model": trained BERTopic model
+      - "topics": topics extracted by the model
+      - "topics_info": information about the topics
+    """
+    # Get texts list and embeddings
+    texts_list = st.session_state["df"][TEXT_COLUMN].tolist()
+    embeddings = get_embeddings(texts_list)
+
+    # If use_tags is True, get tags from dataframe
+    if st.session_state["use_tags"]:
+        # Convert tags to string
+        st.session_state["df"][TAGS_COLUMN] = st.session_state["df"][
+            TAGS_COLUMN
+        ].astype(str)
+
+        # Get zeroshot_topic_list from tags
+        zeroshot_topic_list = (
+            st.session_state["df"][TAGS_COLUMN]
+            .fillna("")
+            .str.findall(r"#\w+")
+            .explode()
+            .unique()
+        )
+
+        # Remove # and _ from tags and convert to string
+        zeroshot_topic_list = [
+            str(tag).replace("#", "").replace("_", " ")
+            for tag in zeroshot_topic_list
+            if tag
+        ]
+    # Else, set zeroshot_topic_list to None
+    else:
+        zeroshot_topic_list = None
+
+    # Train topic model
+    bertopic, topics = fit_bertopic(
+        texts_list,
+        embeddings,
+        st.session_state["min_articles_per_topic"],
+        zeroshot_topic_list=zeroshot_topic_list,
+    )
+
+    # Set session_state
+    st.session_state["topic_model"] = bertopic
+    st.session_state["topics"] = topics
+
+    topic_info = bertopic.get_topic_info()
+    topic_info["Representation"] = topic_info["Representation"].apply(
+        lambda x: x[:TOP_N_WORDS]
+    )
+
+    st.session_state["topics_info"] = topic_info[
+        topic_info["Topic"] != -1
+    ]  # exclude -1 topic from topic list

bertrend_apps/exploration/curebot/tabs/tab2.py

@@ -0,0 +1,95 @@
+import streamlit as st
+
+from bertrend_apps.exploration.curebot.app_utils import (
+    URL_COLUMN,
+    display_representative_documents,
+    display_source_distribution,
+    get_website_name,
+)
+
+
+def show() -> None:
+    # Check if a model is trained
+    if "topic_model" not in st.session_state:
+        st.warning("Veuillez ajouter des données et lancer la détection des sujets.")
+    else:
+        # Show sidebar with topic list
+        show_topic_list()
+
+        # Display selected topic info
+        display_topic_info(st.session_state.get("selected_topic_number", 0))
+
+
+def show_topic_list():
+    """
+    Show topic list in the sidebar.
+    Each topic is a button that, when clicked, sets session_state:
+        - "selected_topic_number": topic number, this topic is displayed in results tab.
+    """
+    with st.sidebar:
+        with st.expander("Sujets détectés", expanded=True):
+            # Topics list
+            for _, topic in st.session_state["topics_info"].iterrows():
+                # Display button for each topic
+                topic_number = topic["Topic"]
+                button_title = topic["llm_description"]
+                st.button(
+                    str(topic_number + 1) + " - " + button_title,
+                    use_container_width=True,
+                    on_click=set_topic_selection,
+                    args=(topic_number,),
+                )
+
+
+def set_topic_selection(topic_number: int):
+    """Set "selected_topic_number" in the session state."""
+    st.session_state["selected_topic_number"] = topic_number
+
+
+def display_topic_info(topic_number: int):
+    """Display "selected_topic_number" associated topic information."""
+    # Filter df to get only the selected topic
+    selected_topic_df = st.session_state["df"][
+        st.session_state["df"]["topics"] == topic_number
+    ]
+
+    # Get topic info
+    docs_count = len(selected_topic_df)
+    key_words = st.session_state["topics_info"].iloc[topic_number]["Representation"]
+    llm_description = st.session_state["topics_info"].iloc[topic_number][
+        "llm_description"
+    ]
+
+    # Display topic info
+    st.write(f"# {llm_description}")
+    st.write(f"### {docs_count} documents")
+    st.markdown(f"### #{' #'.join(key_words)}")
+
+    # Get unique sources
+    sources = selected_topic_df[URL_COLUMN].apply(get_website_name).unique()
+
+    # Multi-select for sources
+    selected_sources = st.multiselect(
+        "Sélectionner les sources à afficher :",
+        options=["All"] + sorted(list(sources)),
+        default=["All"],
+    )
+
+    # Create two columns
+    col21, col22 = st.columns([0.3, 0.7])
+
+    with col21:
+        # Pass the full representative_df to display_source_distribution
+        display_source_distribution(selected_topic_df, selected_sources)
+
+    with col22:
+        # Filter the dataframe only for document display
+        if "All" not in selected_sources:
+            filtered_df = selected_topic_df[
+                selected_topic_df[URL_COLUMN]
+                .apply(get_website_name)
+                .isin(selected_sources)
+            ]
+        else:
+            filtered_df = selected_topic_df
+        display_representative_documents(filtered_df)

bertrend_apps/exploration/curebot/tabs/tab3.py

@@ -0,0 +1,125 @@
+import locale
+from pathlib import Path
+import jinja2
+import streamlit as st
+from bertrend.llm_utils.newsletter_features import md2html
+from bertrend_apps.exploration.curebot.app_utils import (
+    NEWSLETTER_TEMPLATE,
+    create_newsletter,
+)
+
+# Set french locale
+locale.setlocale(locale.LC_ALL, "fr_FR.UTF-8")
+
+CSS_STYLE = (
+    Path(__file__).parent.parent.parent.parent.parent
+    / "bertrend/llm_utils/newsletter.css"
+)
+
+
+def show() -> None:
+    if "topic_model" not in st.session_state:
+        st.warning("Veuillez ajouter des données et lancer la détection des sujets.")
+    else:
+        # Show newsletter parameters
+        newsletter_parameters()
+
+        # Columns for buttons
+        col1, col2, col3 = st.columns([1, 1, 1])
+
+        # Show newsletter creation button
+        with col1:
+            if st.button("Créer la newsletter"):
+                with st.spinner("Création de la newsletter..."):
+                    get_newsletter()
+
+        # Show newsletter edition button
+        with col2:
+            st.button(
+                "Éditer",
+                on_click=edit_newsletter,
+                disabled=not "newsletter_text" in st.session_state,
+            )
+
+        # Show newsletter download button
+        with col3:
+            # If newsletter created, set it as data to download
+            if "newsletter_text" in st.session_state:
+                data = md2html(st.session_state["newsletter_text"], css_style=CSS_STYLE)
+            # Else, set data to empty string
+            else:
+                data = ""
+
+            # Download button
+            st.download_button(
+                "Télécharger",
+                file_name="newsletters.html",
+                mime="text/html",
+                data=data,
+                disabled=not "newsletter_text" in st.session_state,
+            )
+
+        # Show newsletter
+        if "newsletter_text" in st.session_state:
+            st.write(st.session_state["newsletter_text"], unsafe_allow_html=True)
+
+
+def newsletter_parameters() -> None:
+    """
+    Show newsletter parameters:
+    - Number of topics to include in the newsletter
+    - Number of articles per topic to include in the newsletter
+    """
+    col31, col32 = st.columns([1, 1])
+    with col31:
+        # Select number of topics to include in the newsletter
+        st.slider(
+            "Nombre de sujets",
+            1,
+            len(st.session_state["topics_info"]),
+            value=min(len(st.session_state["topics"]), 3),
+            key="newsletter_nb_topics",
+        )
+    with col32:
+        # Select number of articles per topic to include in the newsletter
+        st.slider(
+            "Nombre d'articles par sujet",
+            1,
+            10,
+            value=4,
+            key="newsletter_nb_articles_per_topic",
+        )
+
+
+def get_newsletter() -> None:
+    """
+    Create a newsletter based on the selected topics and articles.
+    Sets in sessions_state:
+    - newsletter_dict: a dictionary containing the newsletter data
+    - newsletter_text: the newsletter text
+    """
+    # Newsletter dict
+    st.session_state["newsletter_dict"] = create_newsletter(
+        st.session_state["df"],
+        st.session_state["topics_info"],
+        st.session_state["newsletter_nb_topics"],
+        st.session_state["newsletter_nb_articles_per_topic"],
+    )
+
+    # Newsletter text
+    template = jinja2.Template(NEWSLETTER_TEMPLATE)
+    st.session_state["newsletter_text"] = template.render(
+        st.session_state["newsletter_dict"]
+    )
+
+
+@st.dialog("Éditer la newsletter", width="large")
+def edit_newsletter() -> None:
+    edited_newsltter = st.text_area(
+        "",
+        value=st.session_state["newsletter_text"],
+        height=500,
+    )
+    st.session_state["newsletter_text"] = edited_newsltter
+    if st.button("Enregistrer", type="primary"):
+        st.rerun()