bertrend 0.3.1__py3-none-any.whl

bertrend/demos/demos_utils/parameters_component.py

@@ -0,0 +1,162 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+import streamlit as st
+
+from code_editor import code_editor
+
+from bertrend import (
+    BERTOPIC_DEFAULT_CONFIG_PATH,
+    BERTREND_DEFAULT_CONFIG_PATH,
+    EMBEDDING_CONFIG,
+    load_toml_config,
+)
+from bertrend.demos.demos_utils.state_utils import (
+    register_widget,
+    save_widget_state,
+    SessionStateManager,
+    register_multiple_widget,
+    reset_widget_state,
+)
+from bertrend.config.parameters import (
+    EMBEDDING_DTYPES,
+    LANGUAGES,
+    ENGLISH_EMBEDDING_MODELS,
+    FRENCH_EMBEDDING_MODELS,
+    REPRESENTATION_MODELS,
+    MMR_REPRESENTATION_MODEL,
+)
+
+from bertrend.demos.demos_utils.icons import INFO_ICON
+
+
+def display_local_embeddings():
+    """UI settings for local embedding service"""
+    register_multiple_widget("language", "embedding_dtype", "embedding_model_name")
+
+    language = st.selectbox(
+        "Select Language",
+        LANGUAGES,
+        key="language",
+        on_change=_on_language_change,
+    )
+    st.selectbox(
+        "Embedding Dtype",
+        EMBEDDING_DTYPES,
+        key="embedding_dtype",
+        on_change=save_widget_state,
+    )
+    embedding_models = (
+        ENGLISH_EMBEDDING_MODELS if language == "English" else FRENCH_EMBEDDING_MODELS
+    )
+    st.selectbox(
+        "Embedding Model",
+        options=embedding_models,
+        key="embedding_model_name",
+        on_change=save_widget_state,
+    )
+
+
+def _on_language_change():
+    save_widget_state()
+    # required to handle multi-page app and restore_state call:
+    reset_widget_state("embedding_model_name")
+    st.session_state.pop("embedding_model_name")
+
+
+def display_remote_embeddings():
+    """UI settings for remote embedding service"""
+    register_widget("embedding_service_url")
+    if "embedding_service_url" not in st.session_state:
+        st.session_state["embedding_service_url"] = EMBEDDING_CONFIG["url"]
+    st.text_input(
+        "Embedding service URL",
+        key="embedding_service_url",
+        on_change=save_widget_state,
+    )
+
+
+def display_embedding_hyperparameters():
+    """UI settings for embedding hyperparameters"""
+    # Embedding model parameters
+    with st.expander("Embedding Model Settings", expanded=False):
+        register_widget("embedding_service_type")
+        if "embedding_service_type" not in st.session_state:
+            st.session_state["embedding_service_type"] = "remote"
+        st.segmented_control(
+            "Embedding service",
+            selection_mode="single",
+            key="embedding_service_type",
+            options=["local", "remote"],
+            on_change=save_widget_state,
+        )
+        if st.session_state["embedding_service_type"] == "local":
+            display_local_embeddings()
+        else:
+            display_remote_embeddings()
+
+
+def display_bertopic_hyperparameters():
+    # BERTopic model parameters
+    with st.expander("BERTopic Model Settings", expanded=False):
+        # If BERTopic config is already in session state, use it
+        if "bertopic_config" in st.session_state:
+            toml_txt = st.session_state["bertopic_config"]
+        # Else get BERTopic default configuration
+        else:
+            with open(BERTOPIC_DEFAULT_CONFIG_PATH, "r") as f:
+                # Load default parameter the first time
+                toml_txt = f.read()
+
+        # Add code editor to edit the config file
+        st.write(INFO_ICON + " CTRL + Enter to update")
+        config_editor = code_editor(
+            toml_txt,
+            lang="toml",
+        )
+
+        # If code is edited, update config
+        if config_editor["text"] != "":
+            st.session_state["bertopic_config"] = config_editor["text"]
+        # Else use default config
+        else:
+            st.session_state["bertopic_config"] = toml_txt
+
+
+def display_bertrend_hyperparameters():
+    """UI settings for Bertrend hyperparameters"""
+    with st.expander("BERTrend Model Settings", expanded=False):
+        # Get BERTrend default configuration
+        with open(BERTREND_DEFAULT_CONFIG_PATH, "r") as f:
+            # Load default parameter the first time
+            toml_txt = f.read()
+
+        # Add code editor to edit the config file
+        st.write(INFO_ICON + " CTRL + Enter to update")
+        config_editor = code_editor(toml_txt, lang="toml")
+
+        # If code is edited, update config
+        if config_editor["text"] != "":
+            st.session_state["bertrend_config"] = config_editor["text"]
+        # Else use default config
+        else:
+            st.session_state["bertrend_config"] = toml_txt
+
+        # Save granularity in session state as it is re-used in other components
+        st.session_state["granularity"] = load_toml_config(
+            st.session_state["bertrend_config"]
+        )["granularity"]
+
+
+def display_representation_model_options():
+    """UI settings for representation model options"""
+    with st.expander("Representation model selection", expanded=False):
+        register_widget("representation_models")
+        selected_models = st.multiselect(
+            label="Select representation models",
+            options=REPRESENTATION_MODELS,
+            default=MMR_REPRESENTATION_MODEL,
+            key="representation_models",
+            on_change=save_widget_state,
+        )

bertrend/demos/demos_utils/state_utils.py

@@ -0,0 +1,99 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+from typing import Any
+
+import numpy as np
+import pandas as pd
+
+import streamlit as st
+
+STATE_KEYS = "state_keys"
+WIDGET_STATE = "widget_state"
+
+# Set of util functions to manage the state of widgets in streamlit
+# - at the beginning of each page: add restore_widget_state() to restore the last selected values
+# - each time a widget is used, call *before* register_widget with the *key* of the widget as argument
+# and use the call_back of the widget (for example on_change) to call save_widget_state
+
+
+def register_widget(key: str):
+    """Keeps track of widget state in case of multi-pages Streamlit app. It shall be called with the key of
+    the widget before the widget creation."""
+    if STATE_KEYS not in st.session_state.keys():
+        st.session_state[STATE_KEYS] = []
+    if key not in st.session_state[STATE_KEYS]:
+        st.session_state[STATE_KEYS].append(key)
+
+
+def reset_widget_state(key: str):
+    """Removes the widget state from session state."""
+    if WIDGET_STATE in st.session_state.keys():
+        st.session_state[WIDGET_STATE].pop(key)
+
+
+def register_multiple_widget(*keys: str):
+    """Keeps track of widget state in case of multi-pages Streamlit app. It shall be called with the key of
+    the widget before the widget creation."""
+    for key in keys:
+        register_widget(key)
+
+
+def save_widget_state():
+    """(Callback) Function to save the widget state in case of multi-pages Streamlit app.
+    It shall be used as the call_back of the widget (for example on_change)."""
+    if STATE_KEYS in st.session_state.keys():
+        st.session_state[WIDGET_STATE] = {
+            key: st.session_state[key]
+            for key in st.session_state[STATE_KEYS]
+            if key in st.session_state
+        }
+
+
+def restore_widget_state():
+    """Function to restore widget states in case of multi-pages Streamlit app.
+    It shall be called at the start of each page."""
+    if WIDGET_STATE in st.session_state.keys():
+        for k, v in st.session_state[WIDGET_STATE].items():
+            st.session_state[k] = v
+
+
+class SessionStateManager:
+    """Class to ease the management of Streamlit session state"""
+
+    @staticmethod
+    def get(key: str, default: Any = None) -> Any:
+        return st.session_state.get(key, default)
+
+    @staticmethod
+    def set(key: str, value: Any) -> None:
+        st.session_state[key] = value
+
+    @staticmethod
+    def get_or_set(key: str, default: Any) -> Any:
+        if key not in st.session_state:
+            st.session_state[key] = default
+        return st.session_state[key]
+
+    @staticmethod
+    def get_multiple(*keys: str) -> dict[str, Any]:
+        return {key: st.session_state.get(key) for key in keys}
+
+    @staticmethod
+    def set_multiple(**kwargs: Any) -> None:
+        for key, value in kwargs.items():
+            st.session_state[key] = value
+
+    @staticmethod
+    def clear() -> None:
+        st.session_state.clear()
+
+    @staticmethod
+    def get_dataframe(key: str) -> pd.DataFrame | None:
+        df = st.session_state.get(key)
+        return df if isinstance(df, pd.DataFrame) else None
+
+    @staticmethod
+    def get_embeddings(key: str = "embeddings") -> np.ndarray | None:
+        return st.session_state.get(key)

bertrend/demos/start_summarizer_app.sh

@@ -0,0 +1,9 @@
+#
+# Copyright (c) 2024, RTE (https://www.rte-france.com)
+# See AUTHORS.txt
+# SPDX-License-Identifier: MPL-2.0
+# This file is part of BERTrend.
+#
+
+# Starts the Summarizer app (a comparison of various method of text summarization)
+CUDA_VISIBLE_DEVICES=0 OPENAI_API_KEY=$AZURE_WATTELSE_OPENAI_API_KEY OPENAI_ENDPOINT=$AZURE_WATTELSE_OPENAI_ENDPOINT OPENAI_DEFAULT_MODEL_NAME=$AZURE_WATTELSE_OPENAI_DEFAULT_MODEL_NAME streamlit run --theme.primaryColor royalblue summarization/summarizer_app.py

bertrend/demos/start_topic_analysis_app.sh

@@ -0,0 +1,9 @@
+#
+# Copyright (c) 2024, RTE (https://www.rte-france.com)
+# See AUTHORS.txt
+# SPDX-License-Identifier: MPL-2.0
+# This file is part of BERTrend.
+#
+
+# Starts the Topic Analysis application
+CUDA_VISIBLE_DEVICES=0 streamlit run --theme.primaryColor royalblue topic_analysis/app.py

bertrend/demos/start_weak_signals_app.sh

@@ -0,0 +1,9 @@
+#
+# Copyright (c) 2024, RTE (https://www.rte-france.com)
+# See AUTHORS.txt
+# SPDX-License-Identifier: MPL-2.0
+# This file is part of BERTrend.
+#
+
+# Starts the Trend Analysis application
+CUDA_VISIBLE_DEVICES=0 streamlit run --theme.primaryColor royalblue weak_signals/app.py

bertrend/demos/summarization/__init__.py

@@ -0,0 +1,4 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.

bertrend/demos/summarization/summarizer_app.py

@@ -0,0 +1,98 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+import os
+
+import streamlit as st
+from loguru import logger
+
+from bertrend.demos.demos_utils.icons import WARNING_ICON
+from bertrend.services.summary.abstractive_summarizer import AbstractiveSummarizer
+from bertrend.services.summary.chatgpt_summarizer import GPTSummarizer
+from bertrend.services.summary.extractive_summarizer import (
+    EnhancedExtractiveSummarizer,
+    ExtractiveSummarizer,
+)
+
+SUMMARIZER_OPTIONS_MAPPER = {
+    "GPTSummarizer": GPTSummarizer,
+    "EnhancedExtractiveSummarizer": EnhancedExtractiveSummarizer,
+    "ExtractiveSummarizer": ExtractiveSummarizer,
+    "AbstractiveSummarizer": AbstractiveSummarizer,
+}
+
+DEFAULT_TEXT = """RTE, sigle du Réseau de transport d'électricité, est le gestionnaire de réseau de transport français responsable du réseau public de transport d'électricité haute tension en France métropolitaine (la Corse n'est pas gérée par RTE). Sa mission fondamentale est d’assurer à tous ses clients l’accès à une alimentation électrique économique, sûre et propre. RTE connecte ses clients par une infrastructure adaptée et leur fournit tous les outils et services qui leur permettent d’en tirer parti pour répondre à leurs besoins. À cet effet, RTE exploite, maintient et développe le réseau à haute et très haute tension. Il est le garant du bon fonctionnement et de la sûreté du système électrique. RTE achemine l’électricité entre les fournisseurs d’électricité (français et européens) et les consommateurs, qu’ils soient distributeurs d’électricité ou industriels directement raccordés au réseau de transport. Plus de 105 000 km de lignes comprises entre 45 000 et 400 000 volts et 50 lignes transfrontalières connectent le réseau français à 33 pays européens, offrant ainsi des opportunités d’échanges d’électricité essentiels pour l’optimisation économique du système électrique.
+
+RTE fait partie du Réseau européen des gestionnaires de réseau de transport d’électricité (ENTSO-E), organisation qui regroupe les gestionnaires de réseaux de transport à haute et très haute tension de 35 pays3. Elle découle de la fusion le 1er juillet 2009 de l'UCTE, de BALTSO, de NORDEL, d'ATSOI et d'UKTSOA.
+
+Les réseaux de ces sociétés desservent, via les réseaux de distribution, une population d'environ 500 millions de personnes. Ils se décomposent en cinq grands systèmes synchrones : l'Europe continentale (à laquelle est rattachée la Turquie), les pays baltes, les pays nordiques, l'Irlande et la Grande-Bretagne.
+
+Les lignes à basse et moyenne tension françaises ne sont pas du ressort de RTE. Elles sont essentiellement exploitées par Enedis (anciennement ERDF, filiale de distribution électrique d'EDF), mais aussi par d'autres entreprises locales de distribution (ELD) comme Électricité de Strasbourg, l'Usine d'électricité de Metz, ou encore Gascogne Énergies Services à Aire-sur-l’Adour dans les Landes."""
+
+models = {}
+
+
+@st.cache_resource
+def get_summarizer(summary_model):
+    """Instantiates models once, keep them in memory"""
+    kwargs = {}
+    if summary_model == "GPTSummarizer":
+        kwargs["api_key"] = st.session_state.openai_api_key
+        kwargs["endpoint"] = st.session_state.openai_endpoint
+
+    summarizer_class = SUMMARIZER_OPTIONS_MAPPER[summary_model]
+    return summarizer_class(**kwargs)
+
+
+def app():
+    st.title("Comparison of summarizers")
+
+    st.warning(
+        "Warning: be sure that when using a GPT summarizer, the server is on Azure or local. "
+        "Otherwise, use it EXCLUSIVELY on public data.",
+        icon=WARNING_ICON,
+    )
+
+    summary_model = st.selectbox(
+        "summary model", SUMMARIZER_OPTIONS_MAPPER.keys(), key="summary_model"
+    )
+    summary_ratio = (
+        st.number_input(
+            "summary ratio", min_value=1, max_value=100, value=20, key="summary_ratio"
+        )
+        / 100
+    )
+    st.text_input(
+        "Openai API key",
+        key="openai_api_key",
+        value=os.getenv("OPENAI_API_KEY"),
+        type="password",
+    )
+    st.text_input(
+        "Openai endpoint", key="openai_endpoint", value=os.getenv("OPENAI_ENDPOINT")
+    )
+    st.text_input(
+        "Openai model name",
+        key="openai_model_name",
+        value=os.getenv("OPENAI_DEFAULT_MODEL_NAME"),
+    )
+
+    col1, col2 = st.columns(2)
+    with col1:
+        st.text_area("Input text", DEFAULT_TEXT, height=250, key="text")
+    with col2:
+        st.text_area("Summary", "", height=250, key="summary")
+
+    def on_click():
+        summarizer = get_summarizer(summary_model)
+        st.session_state.summary = summarizer.generate_summary(
+            st.session_state.text, max_length_ratio=summary_ratio
+        )
+        logger.debug(st.session_state.summary)
+
+    st.button("Summarize", on_click=on_click)
+
+
+if __name__ == "__main__":
+    app()

bertrend/demos/topic_analysis/__init__.py

@@ -0,0 +1,4 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.

bertrend/demos/topic_analysis/app.py

@@ -0,0 +1,87 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+
+import torch
+
+# workaround with streamlit to avoid errors Examining the path of torch.classes raised: Tried to instantiate class 'path.path’, but it does not exist! Ensure that it is registered via torch::class
+torch.classes.__path__ = []
+
+import locale
+
+import streamlit as st
+
+from bertrend.demos.demos_utils.icons import (
+    SAVE_ICON,
+    TOPIC_EXPLORATION_ICON,
+    TOPIC_VISUALIZATION_ICON,
+    TEMPORAL_VISUALIZATION_ICON,
+    NEWSLETTER_ICON,
+)
+from bertrend.demos.demos_utils.state_utils import restore_widget_state
+
+TITLE = "BERTrend - Topic Analysis demo"
+LAYOUT = "wide"
+
+
+# Set locale for French date names
+locale.setlocale(locale.LC_TIME, "fr_FR.UTF-8")
+
+
+def define_pages():
+    data_page = st.Page(
+        page="demo_pages/training_page.py",
+        title="Data loading & model training",
+        icon=SAVE_ICON,
+    )
+    topic_exploration_page = st.Page(
+        page="demo_pages/explore_topics.py",
+        title="Topic exploration",
+        icon=TOPIC_EXPLORATION_ICON,
+    )
+    topic_visualization_page = st.Page(
+        page="demo_pages/topic_visualizations.py",
+        title="Topic visualization",
+        icon=TOPIC_VISUALIZATION_ICON,
+    )
+    temporal_visualization_page = st.Page(
+        page="demo_pages/temporal_visualizations.py",
+        title="Temporal visualization",
+        icon=TEMPORAL_VISUALIZATION_ICON,
+    )
+    newsletter_page = st.Page(
+        page="demo_pages/newsletters_generation.py",
+        title="Newsletter generation",
+        icon=NEWSLETTER_ICON,
+    )
+
+    pg = st.navigation(
+        {
+            "Topic Analysis": [
+                data_page,
+                topic_exploration_page,
+                topic_visualization_page,
+                temporal_visualization_page,
+            ],
+            "Application example": [newsletter_page],
+        }
+    )
+    return pg
+
+
+def main():
+    pg = define_pages()
+    st.set_page_config(
+        page_title=TITLE,
+        layout=LAYOUT,
+        initial_sidebar_state="expanded",
+        page_icon=":part_alternation_mark:",
+    )
+    # Restore widget state
+    restore_widget_state()
+    pg.run()
+
+
+if __name__ == "__main__":
+    main()

bertrend/demos/topic_analysis/app_utils.py

@@ -0,0 +1,69 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+import numpy as np
+import pandas as pd
+import streamlit as st
+from bertopic import BERTopic
+
+from bertrend.utils.data_loading import (
+    TIMESTAMP_COLUMN,
+    GROUPED_TIMESTAMP_COLUMN,
+    TEXT_COLUMN,
+    TITLE_COLUMN,
+    URL_COLUMN,
+    CITATION_COUNT_COL,
+)
+
+
+def _make_dynamic_topics_split(df: pd.DataFrame, nr_bins: int):
+    """
+    Split docs into nr_bins and generate a llm_utils timestamp label into a new column
+    """
+    df = df.sort_values(TIMESTAMP_COLUMN, ascending=False)
+    split_df = np.array_split(df, nr_bins)
+    for split in split_df:
+        split[GROUPED_TIMESTAMP_COLUMN] = split[TIMESTAMP_COLUMN].max()
+    return pd.concat(split_df)
+
+
+@st.cache_data
+def compute_topics_over_time(
+    _topic_model: BERTopic,
+    df: pd.DataFrame,
+    nr_bins: int,
+):
+    """Computes topics over time."""
+    df = _make_dynamic_topics_split(df, nr_bins)
+    res = _topic_model.topics_over_time(
+        df[TEXT_COLUMN],
+        df[GROUPED_TIMESTAMP_COLUMN],
+        global_tuning=False,
+    )
+    return res
+
+
+def print_docs_for_specific_topic(
+    df: pd.DataFrame, topics: list[int], topic_number: int
+):
+    """
+    Print documents for a specific topic
+    """
+    columns_list = [
+        col
+        for col in [
+            TITLE_COLUMN,
+            TEXT_COLUMN,
+            URL_COLUMN,
+            TIMESTAMP_COLUMN,
+            CITATION_COUNT_COL,
+        ]
+        if col in df.keys()
+    ]
+    df = df.loc[pd.Series(topics) == topic_number][columns_list]
+    for _, doc in df.iterrows():
+        st.write(f"[{doc.title}]({doc.url})")
+
+
+# TODO: Remove "put embeddings in cache" option since it's unadvised due to the large size of embeddings returned by embedding model (sentence and token embeddings)

bertrend/demos/topic_analysis/demo_pages/__init__.py

@@ -0,0 +1,4 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.