bertrend 0.3.1__py3-none-any.whl

bertrend_apps/prospective_demo/dashboard_common.py

@@ -0,0 +1,79 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+
+import uuid
+from typing import Any
+
+import pandas as pd
+import streamlit as st
+
+from bertrend.demos.demos_utils.icons import WARNING_ICON
+from bertrend_apps.prospective_demo import NOISE, WEAK_SIGNALS, STRONG_SIGNALS
+from bertrend_apps.prospective_demo.models_info import get_models_info
+
+COLS_RATIO_ID_TS = [2 / 7, 5 / 7]
+
+
+@st.cache_data
+def get_df_topics(model_interpretation_path=None) -> dict[str, pd.DataFrame]:
+    dfs_topics = {}
+    for df_id in [NOISE, WEAK_SIGNALS, STRONG_SIGNALS]:
+        df_path = model_interpretation_path / f"{df_id}.parquet"
+        dfs_topics[df_id] = (
+            pd.read_parquet(df_path) if df_path.exists() else pd.DataFrame()
+        )
+    return dfs_topics
+
+
+def update_key(key: str, new_value: Any):
+    st.session_state[key] = new_value
+    # reset ts value in order to avoid errors if a previous ts value is not available for the new key value
+
+
+def update_key_and_ts(key: str, new_value: Any):
+    update_key(key, new_value)
+    # reset ts value in order to avoid errors if a previous ts value is not available for the new key value
+    if "reference_ts" in st.session_state:
+        del st.session_state["reference_ts"]
+
+
+def choose_id_and_ts():
+    col1, col2 = st.columns(COLS_RATIO_ID_TS)
+    with col1:
+        options = sorted(st.session_state.user_feeds.keys())
+        if "model_id" not in st.session_state:
+            st.session_state.model_id = options[0]
+        model_id_key = uuid.uuid4()
+        model_id = st.selectbox(
+            "Sélection de la veille",
+            options=options,
+            index=options.index(st.session_state.model_id),
+            key=model_id_key,  # to avoid pb of unicity if displayed on several places
+            on_change=lambda: update_key_and_ts(
+                "model_id", st.session_state[model_id_key]
+            ),
+        )
+    with col2:
+        list_models = get_models_info(model_id)
+        if not list_models:
+            st.warning(f"{WARNING_ICON} Pas de modèle disponible")
+            st.stop()
+        elif len(list_models) < 2:
+            st.warning(
+                f"{WARNING_ICON} 2 modèles minimum pour analyser les tendances !"
+            )
+            st.stop()
+        if "reference_ts" not in st.session_state:
+            st.session_state.reference_ts = list_models[-1]
+        ts_key = uuid.uuid4()
+        st.select_slider(
+            "Date d'analyse",
+            options=list_models,
+            value=st.session_state.reference_ts,
+            format_func=lambda ts: ts.strftime("%d/%m/%Y"),
+            help="Sélection de la date d'analyse parmi celles disponibles",
+            key=ts_key,  # to avoid pb of unicity if displayed on several places
+            on_change=lambda: update_key("reference_ts", st.session_state[ts_key]),
+        )

bertrend_apps/prospective_demo/dashboard_signals.py

@@ -0,0 +1,226 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+import pandas as pd
+import streamlit as st
+
+from bertrend.demos.demos_utils.icons import (
+    WARNING_ICON,
+    WEAK_SIGNAL_ICON,
+    STRONG_SIGNAL_ICON,
+    NOISE_ICON,
+)
+from bertrend_apps.prospective_demo import (
+    LLM_TOPIC_DESCRIPTION_COLUMN,
+    NOISE,
+    WEAK_SIGNALS,
+    STRONG_SIGNALS,
+    get_model_interpretation_path,
+    LLM_TOPIC_TITLE_COLUMN,
+    URLS_COLUMN,
+)
+from bertrend_apps.prospective_demo.dashboard_common import (
+    choose_id_and_ts,
+    get_df_topics,
+)
+from bertrend_apps.prospective_demo.i18n import translate
+
+COLS_RATIO = [4 / 7, 3 / 7]
+
+
+def signal_analysis():
+    # ID and timestamp selection
+    choose_id_and_ts()
+    model_id = st.session_state.model_id
+    reference_ts = st.session_state.reference_ts
+
+    model_interpretation_path = get_model_interpretation_path(
+        user_name=st.session_state.username,
+        model_id=model_id,
+        reference_ts=reference_ts,
+    )
+
+    # TODO: decide which column to display
+    columns = [
+        "Topic",
+        LLM_TOPIC_TITLE_COLUMN,
+        LLM_TOPIC_DESCRIPTION_COLUMN,
+        # "Representation",
+        URLS_COLUMN,
+        "Latest_Popularity",
+        "Docs_Count",
+        # "Paragraphs_Count",
+        "Documents",
+        "Sources",
+        "Source_Diversity",
+        "Latest_Timestamp",
+    ]
+    column_config = {
+        "Topic": st.column_config.NumberColumn(
+            "Topic",
+            pinned=True,
+        ),
+        LLM_TOPIC_TITLE_COLUMN: st.column_config.TextColumn(
+            translate("title"), pinned=True, width="large"
+        ),
+        "Latest_Popularity": st.column_config.ProgressColumn(
+            format="%i",
+            max_value=50,
+        ),
+        "Source_Diversity": st.column_config.ProgressColumn(
+            format="%i",
+            max_value=50,
+        ),
+        "Latest_Timestamp": st.column_config.DateColumn(
+            format="DD/MM/YYYY",
+        ),
+        URLS_COLUMN: st.column_config.LinkColumn(),
+    }
+
+    dfs_topics = get_df_topics(model_interpretation_path)
+
+    col1, col2 = st.columns(COLS_RATIO)
+    with col1:
+        # Display dataframes for weak_signals, strong, etc
+        display_translated_signal_categories(
+            dfs_topics[NOISE],
+            dfs_topics[WEAK_SIGNALS],
+            dfs_topics[STRONG_SIGNALS],
+            reference_ts,
+            columns=columns,
+            column_config=column_config,
+        )
+
+    with col2:
+        st.info(translate("todo_message"))
+        explore_topic_sources(dfs_topics)
+
+
+@st.fragment
+def explore_topic_sources(dfs_topics):
+    st.write(f"**{translate('explore_sources_by_topic')}**")
+    selected_signal_type = st.pills(
+        translate("signal_type"),
+        label_visibility="hidden",
+        options=[translate("emerging_topics"), translate("strong_topics")],
+        selection_mode="single",
+        default=translate("emerging_topics"),
+    )
+    if selected_signal_type == translate("strong_topics"):
+        selected_df = dfs_topics.get(STRONG_SIGNALS)
+    else:
+        selected_df = dfs_topics.get(WEAK_SIGNALS)
+    if selected_df is None or selected_df.empty:
+        st.warning(f"{WARNING_ICON} {translate('no_data')}")
+    else:
+        selected_df = selected_df.sort_values(by=["Latest_Popularity"], ascending=False)
+        options = selected_df["Topic"].tolist()
+        topic_id = st.selectbox(
+            index=None,
+            label=translate("topic_selection"),
+            label_visibility="hidden",
+            options=options,
+            format_func=lambda x: f"{translate('topic')} {x}: "
+            + selected_df[selected_df["Topic"] == x][LLM_TOPIC_TITLE_COLUMN].values[0],
+        )
+        if topic_id is None:
+            return
+        if selected_signal_type == translate("strong_topics"):
+            color = "green"
+        else:
+            color = "orange"
+        row = selected_df[selected_df["Topic"] == topic_id]
+        display_topic_links(
+            title=f":{color}[**{row[LLM_TOPIC_TITLE_COLUMN].values[0]}**]",
+            desc=row[LLM_TOPIC_DESCRIPTION_COLUMN].values[0],
+            df=list(selected_df.query(f"Topic == {topic_id}")[URLS_COLUMN])[0],
+        )
+
+
+def display_translated_signal_categories(
+    noise_topics_df: pd.DataFrame,
+    weak_signal_topics_df: pd.DataFrame,
+    strong_signal_topics_df: pd.DataFrame,
+    window_end: pd.Timestamp,
+    columns=None,
+    column_order=None,
+    column_config=None,
+):
+    """Wrapper around display_signal_categories_df that uses translated text."""
+    # Weak Signals
+    with st.expander(
+        f":orange[{WEAK_SIGNAL_ICON} {translate('weak_signals')}]", expanded=True
+    ):
+        st.subheader(f":orange[{translate('weak_signals')}]")
+        if not weak_signal_topics_df.empty:
+            displayed_df = weak_signal_topics_df[columns].sort_values(
+                by=["Latest_Popularity"], ascending=False
+            )
+            displayed_df["Documents"] = displayed_df["Documents"].astype(str)
+            st.dataframe(
+                displayed_df,
+                column_order=column_order if column_order else columns,
+                column_config=column_config,
+                hide_index=True,
+            )
+        else:
+            st.info(
+                translate("no_weak_signals").format(timestamp=window_end),
+                icon=WARNING_ICON,
+            )
+
+    # Strong Signals
+    with st.expander(
+        f":green[{STRONG_SIGNAL_ICON} {translate('strong_signals')}]", expanded=True
+    ):
+        st.subheader(f":green[{translate('strong_signals')}]")
+        if not strong_signal_topics_df.empty:
+            displayed_df = strong_signal_topics_df[columns].sort_values(
+                by=["Latest_Popularity"], ascending=False
+            )
+            displayed_df["Documents"] = displayed_df["Documents"].astype(str)
+            st.dataframe(
+                displayed_df,
+                column_order=column_order if column_order else columns,
+                column_config=column_config,
+                hide_index=True,
+            )
+        else:
+            st.info(
+                translate("no_strong_signals").format(timestamp=window_end),
+                icon=WARNING_ICON,
+            )
+
+    # Noise
+    with st.expander(f":grey[{NOISE_ICON} {translate('noise')}]", expanded=True):
+        st.subheader(f":grey[{translate('noise')}]")
+        if not noise_topics_df.empty:
+            displayed_df = noise_topics_df[columns].sort_values(
+                by=["Latest_Popularity"], ascending=False
+            )
+            displayed_df["Documents"] = displayed_df["Documents"].astype(str)
+            st.dataframe(
+                displayed_df,
+                column_order=column_order if column_order else columns,
+                column_config=column_config,
+                hide_index=True,
+            )
+        else:
+            st.info(
+                translate("no_noise_signals").format(timestamp=window_end),
+                icon=WARNING_ICON,
+            )
+
+
+@st.dialog(translate("explore_sources"), width="large")
+def display_topic_links(title: str, desc: str, df: pd.DataFrame):
+    st.subheader(title)
+    st.write(desc)
+    st.dataframe(
+        df,
+        use_container_width=True,
+        column_config={
+            "value": st.column_config.LinkColumn(translate("reference_articles")),
+        },
+    )

bertrend_apps/prospective_demo/feeds_common.py

@@ -0,0 +1,38 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+from pathlib import Path
+
+from loguru import logger
+
+from bertrend import FEED_BASE_PATH, load_toml_config
+from bertrend_apps.prospective_demo import CONFIG_FEEDS_BASE_PATH
+
+
+def read_user_feeds(username: str) -> tuple[dict[str, dict], dict[str, Path]]:
+    """Read user feed config files"""
+    user_feed_dir = CONFIG_FEEDS_BASE_PATH / username
+    user_feed_dir.mkdir(parents=True, exist_ok=True)
+    logger.debug(f"Reading user feeds from: {user_feed_dir}")
+    matching_files = user_feed_dir.rglob("*_feed.toml")
+
+    user_feeds = {}
+    feed_files = {}
+    for f in matching_files:
+        feed_id = f.name.split("_feed.toml")[0]
+        user_feeds[feed_id] = load_toml_config(f)
+        feed_files[feed_id] = f
+
+    return user_feeds, feed_files
+
+
+def get_all_files_for_feed(user_feeds: dict[str, dict], feed_id: str) -> list[Path]:
+    """Returns the paths of all files associated to a feed for the current user."""
+    feed_base_dir = user_feeds[feed_id]["data-feed"]["feed_dir_path"]
+    list_all_files = list(
+        Path(FEED_BASE_PATH, feed_base_dir).glob(
+            f"*{user_feeds[feed_id]['data-feed'].get('id')}*.jsonl*"
+        )
+    )
+    return list_all_files

bertrend_apps/prospective_demo/feeds_config.py

@@ -0,0 +1,261 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+import re
+import time
+from pathlib import Path
+
+import pandas as pd
+import streamlit as st
+import toml
+from loguru import logger
+
+from bertrend.config.parameters import LANGUAGES
+from bertrend.demos.demos_utils.icons import (
+    INFO_ICON,
+    ERROR_ICON,
+    ADD_ICON,
+    EDIT_ICON,
+    DELETE_ICON,
+    WARNING_ICON,
+    TOGGLE_ON_ICON,
+    TOGGLE_OFF_ICON,
+)
+from bertrend_apps.common.crontab_utils import (
+    get_understandable_cron_description,
+    check_if_scrapping_active_for_user,
+    remove_scrapping_for_user,
+    schedule_scrapping,
+)
+from bertrend_apps.data_provider import URL_PATTERN
+from bertrend_apps.prospective_demo.feeds_common import (
+    read_user_feeds,
+)
+from bertrend_apps.prospective_demo import CONFIG_FEEDS_BASE_PATH
+from bertrend_apps.prospective_demo.models_info import (
+    remove_scheduled_training_for_user,
+)
+from bertrend_apps.prospective_demo.streamlit_utils import clickable_df
+
+# Default feed configs
+DEFAULT_GNEWS_CRONTAB_EXPRESSION = "1 0 * * 1"
+DEFAULT_CUREBOT_CRONTAB_EXPRESSION = "42 0,6,12,18 * * *"  # 4 times a day
+DEFAULT_MAX_RESULTS = 20
+DEFAULT_NUMBER_OF_DAYS = 14
+FEED_SOURCES = ["google", "curebot"]
+TRANSLATION = {"English": "Anglais", "French": "Français"}
+
+
+@st.dialog("Configuration d'un nouveau flux de données")
+def edit_feed_monitoring(config: dict | None = None):
+    """Create or update a feed monitoring configuration."""
+    chosen_id = st.text_input(
+        "ID :red[*]",
+        help="Identifiant du flux de données",
+        value=None if not config else config["id"],
+    )
+
+    provider = st.segmented_control(
+        "Source",
+        selection_mode="single",
+        options=FEED_SOURCES,
+        default=FEED_SOURCES[0] if not config else config["provider"],
+        help="Sélection de la source de données",
+    )
+    if provider == "google":
+        query = st.text_input(
+            "Requête :red[*]",
+            value="" if not config else config["query"],
+            help="Saisir ici la requête qui sera faite sur Google News",
+        )
+        language = st.segmented_control(
+            "Langue",
+            selection_mode="single",
+            options=LANGUAGES,
+            default=LANGUAGES[0],
+            format_func=lambda lang: TRANSLATION[lang],
+            help="Choix de la langue",
+        )
+        if "update_frequency" not in st.session_state:
+            st.session_state.update_frequency = (
+                DEFAULT_GNEWS_CRONTAB_EXPRESSION
+                if not config
+                else config["update_frequency"]
+            )
+        new_freq = st.text_input(
+            f"Fréquence d'exécution",
+            value=st.session_state.update_frequency,
+            help=f"Fréquence de collecte des données",
+        )
+        st.session_state.update_frequency = new_freq
+        st.write(display_crontab_description(st.session_state.update_frequency))
+
+    elif provider == "curebot":
+        query = st.text_input(
+            "ATOM feed :red[*]",
+            value="" if not config else config["query"],
+            help="URL du flux de données Curebot",
+        )
+
+    try:
+        get_understandable_cron_description(st.session_state.update_frequency)
+        valid_cron = True
+    except:
+        valid_cron = False
+
+    if st.button(
+        "OK",
+        disabled=not chosen_id
+        or not query
+        or (query and provider == "curebot" and not re.match(URL_PATTERN, query)),
+    ):
+        if not config:
+            config = {}
+        config["id"] = "feed_" + chosen_id
+        config["feed_dir_path"] = (
+            "users/" + st.session_state.username + "/feed_" + chosen_id
+        )
+        config["query"] = query
+        config["provider"] = provider
+        if not config.get("max_results"):
+            config["max_results"] = DEFAULT_MAX_RESULTS
+        if not config.get("number_of_days"):
+            config["number_of_days"] = DEFAULT_NUMBER_OF_DAYS
+        if provider == "google":
+            config["language"] = "fr" if language == "French" else "en"
+            config["update_frequency"] = (
+                st.session_state.update_frequency
+                if valid_cron
+                else DEFAULT_GNEWS_CRONTAB_EXPRESSION
+            )
+        elif provider == "curebot":
+            config["language"] = "fr"
+            config["update_frequency"] = DEFAULT_CUREBOT_CRONTAB_EXPRESSION
+
+        if "update_frequency" in st.session_state:
+            del st.session_state["update_frequency"]  # to avoid memory effect
+
+        # Remove prevous crontab if any
+        remove_scrapping_for_user(feed_id=chosen_id, user=st.session_state.username)
+
+        # Save feed config and update crontab
+        save_feed_config(chosen_id, config)
+
+
+def save_feed_config(chosen_id, feed_config: dict):
+    """Save the feed configuration to disk as a TOML file."""
+    feed_path = (
+        CONFIG_FEEDS_BASE_PATH / st.session_state.username / f"{chosen_id}_feed.toml"
+    )
+    # Save the dictionary to a TOML file
+    with open(feed_path, "w") as toml_file:
+        toml.dump({"data-feed": feed_config}, toml_file)
+    logger.debug(f"Saved feed config {feed_config} to {feed_path}")
+    schedule_scrapping(feed_path, user=st.session_state.username)
+    st.rerun()
+
+
+def display_crontab_description(crontab_expr: str) -> str:
+    try:
+        return f":blue[{INFO_ICON} {get_understandable_cron_description(crontab_expr)}]"
+    except Exception:
+        return f":red[{ERROR_ICON} Expression mal écrite !]"
+
+
+def configure_information_sources():
+    """Configure Information Sources."""
+    # if "user_feeds" not in st.session_state:
+    st.session_state.user_feeds, st.session_state.feed_files = read_user_feeds(
+        st.session_state.username
+    )
+
+    displayed_list = []
+    for k, v in st.session_state.user_feeds.items():
+        displayed_list.append(
+            {
+                "id": k,
+                "provider": v["data-feed"]["provider"],
+                "query": v["data-feed"]["query"],
+                "language": v["data-feed"]["language"],
+                "update_frequency": v["data-feed"]["update_frequency"],
+            }
+        )
+    df = pd.DataFrame(displayed_list)
+    if not df.empty:
+        df = df.sort_values(by="id", inplace=False).reset_index(drop=True)
+
+    if st.button(f":green[{ADD_ICON}]", type="tertiary", help="Nouveau flux de veille"):
+        edit_feed_monitoring()
+
+    clickable_df_buttons = [
+        (EDIT_ICON, edit_feed_monitoring, "secondary"),
+        (lambda x: toggle_icon(df, x), toggle_feed, "secondary"),
+        (DELETE_ICON, handle_delete, "primary"),
+    ]
+    clickable_df(df, clickable_df_buttons)
+
+
+def toggle_icon(df: pd.DataFrame, index: int) -> str:
+    """Switch the toggle icon depending on the statis of the scrapping feed in the crontab"""
+    feed_id = df["id"][index]
+    return (
+        f":green[{TOGGLE_ON_ICON}]"
+        if check_if_scrapping_active_for_user(
+            feed_id=feed_id, user=st.session_state.username
+        )
+        else f":red[{TOGGLE_OFF_ICON}]"
+    )
+
+
+def toggle_feed(cfg: dict):
+    """Activate / deactivate the feed from the crontab"""
+    feed_id = cfg["id"]
+    if check_if_scrapping_active_for_user(
+        feed_id=feed_id, user=st.session_state.username
+    ):
+        if remove_scrapping_for_user(feed_id=feed_id, user=st.session_state.username):
+            st.toast(f"Le flux **{feed_id}** est déactivé !", icon=INFO_ICON)
+            logger.info(f"Flux {feed_id} désactivé !")
+    else:
+        schedule_scrapping(
+            st.session_state.feed_files[feed_id], user=st.session_state.username
+        )
+        st.toast(f"Le flux **{feed_id}** est activé !", icon=WARNING_ICON)
+        logger.info(f"Flux {feed_id} activé !")
+    time.sleep(0.2)
+    st.rerun()
+
+
+def delete_feed_config(feed_id: str):
+    # remove config file
+    file_path: Path = st.session_state.feed_files[feed_id]
+    try:
+        file_path.unlink()
+        logger.debug(f"Feed file {file_path} has been removed.")
+    except Exception as e:
+        logger.error(f"An error occurred: {e}")
+
+
+@st.dialog("Confirmation")
+def handle_delete(row_dict: dict):
+    """Function to handle remove click events"""
+    feed_id = row_dict["id"]
+    st.write(
+        f":orange[{WARNING_ICON}] Voulez-vous vraiment supprimer le flux de veille **{feed_id}** ?"
+    )
+    col1, col2, _ = st.columns([2, 2, 8])
+    with col1:
+        if st.button("Oui", type="primary"):
+            remove_scrapping_for_user(feed_id=feed_id, user=st.session_state.username)
+            delete_feed_config(feed_id)
+            logger.info(f"Flux {feed_id} supprimé !")
+            # Remove from crontab associated training
+            remove_scheduled_training_for_user(
+                model_id=feed_id, user=st.session_state.username
+            )
+            time.sleep(0.2)
+            st.rerun()
+    with col2:
+        if st.button("Non"):
+            st.rerun()

bertrend_apps/prospective_demo/feeds_data.py

@@ -0,0 +1,95 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+import datetime
+from pathlib import Path
+
+import pandas as pd
+import streamlit as st
+
+from bertrend import FEED_BASE_PATH
+from bertrend.utils.data_loading import (
+    load_data,
+    TIMESTAMP_COLUMN,
+    TITLE_COLUMN,
+    URL_COLUMN,
+    TEXT_COLUMN,
+)
+from bertrend_apps.prospective_demo.feeds_common import get_all_files_for_feed
+
+
+def display_data_status():
+    if not st.session_state.user_feeds:
+        return
+
+    col1, col2 = st.columns(2)
+    with col1:
+        st.selectbox(
+            "Sélection de la veille",
+            options=sorted(st.session_state.user_feeds.keys()),
+            key="id_data",
+        )
+
+    with col2:
+        if "data_time_window" not in st.session_state:
+            st.session_state.data_time_window = 7
+        st.slider(
+            "Fenêtre temporelle (jours)",
+            min_value=1,
+            max_value=60,
+            step=1,
+            key="data_time_window",
+        )
+
+    display_data_info_for_feed(st.session_state.id_data)
+
+
+def display_data_info_for_feed(feed_id: str):
+    all_files = get_all_files_for_feed(st.session_state.user_feeds, feed_id)
+    df = get_all_data(files=all_files)
+
+    if df.empty:
+        df_filtered = pd.DataFrame()
+    else:
+        df = df[
+            [TITLE_COLUMN, URL_COLUMN, TEXT_COLUMN, TIMESTAMP_COLUMN]
+        ]  # filter useful columns
+
+        cutoff_date = datetime.datetime.now() - datetime.timedelta(
+            days=st.session_state.data_time_window
+        )
+        df_filtered = df[df[TIMESTAMP_COLUMN] >= cutoff_date]
+
+    stats = {
+        "ID": feed_id,
+        "# Fichiers": len(all_files),
+        "Date début": df[TIMESTAMP_COLUMN].min() if not df.empty else None,
+        "Date fin": df[TIMESTAMP_COLUMN].max() if not df.empty else None,
+        "# Articles": len(df),
+        f"# Articles ({st.session_state.data_time_window} derniers jours)": len(
+            df_filtered
+        ),
+    }
+
+    st.dataframe(pd.DataFrame([stats]))
+
+    st.write(f"#### Données des derniers {st.session_state.data_time_window} jours")
+    st.dataframe(
+        df_filtered,
+        use_container_width=True,
+        hide_index=True,
+        column_config={"url": st.column_config.LinkColumn("url")},
+    )
+
+
+@st.cache_data
+def get_all_data(files: list[Path]) -> pd.DataFrame:
+    """Returns the data contained in the provided files as a single DataFrame."""
+    if not files:
+        return pd.DataFrame()
+    dfs = [load_data(Path(f)) for f in files]
+    new_df = pd.concat(dfs).drop_duplicates(
+        subset=["title"], keep="first", inplace=False
+    )
+    return new_df