bertrend 0.3.1__py3-none-any.whl

bertrend_apps/common/crontab_utils.py

@@ -0,0 +1,113 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+import os
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+from cron_descriptor import (
+    Options,
+    CasingTypeEnum,
+    ExpressionDescriptor,
+    DescriptionTypeEnum,
+)
+from loguru import logger
+
+from bertrend import BEST_CUDA_DEVICE, BERTREND_LOG_PATH, load_toml_config
+
+
+def get_understandable_cron_description(cron_expression: str) -> str:
+    """Returns a human understandable crontab description."""
+    options = Options()
+    options.casing_type = CasingTypeEnum.Sentence
+    options.use_24hour_time_format = True
+    options.locale_code = "fr_FR"
+    descriptor = ExpressionDescriptor(cron_expression, options)
+    return descriptor.get_description(DescriptionTypeEnum.FULL)
+
+
+def add_job_to_crontab(schedule, command, env_vars="") -> bool:
+    """Add the specified job to the crontab."""
+    logger.debug(f"Adding to crontab: {schedule} {command}")
+    home = os.getenv("HOME")
+    # Create crontab, add command - NB: we use the .bashrc to source all environment variables that may be required by the command
+    cmd = f'(crontab -l; echo "{schedule} umask 002; source {home}/.bashrc; {env_vars} {command}" ) | crontab -'
+    returned_value = subprocess.call(cmd, shell=True)  # returns the exit code in unix
+    return returned_value == 0
+
+
+def check_cron_job(pattern: str) -> bool:
+    """Check if a specific pattern (expressed as a regular expression) matches crontab entries."""
+    try:
+        # Run `crontab -l` and capture the output
+        result = subprocess.run(
+            ["crontab", "-l"], capture_output=True, text=True, check=True
+        )
+
+        # Search for the regex pattern in the crontab output
+        if re.search(pattern, result.stdout):
+            return True
+        else:
+            return False
+    except subprocess.CalledProcessError:
+        # If crontab fails (e.g., no crontab for the user), return False
+        return False
+
+
+def remove_from_crontab(pattern: str) -> bool:
+    """Removes from the crontab the job matching the provided pattern (expressed as a regular expression)"""
+    if not (check_cron_job(pattern)):
+        logger.warning("No job matching the provided pattern")
+        return False
+    try:
+        # Retrieve current crontab
+        output = subprocess.check_output(
+            f"crontab -l | grep -Ev {pattern} | crontab -", shell=True
+        )
+        return output == 0
+    except subprocess.CalledProcessError:
+        return False
+
+
+def schedule_scrapping(feed_cfg: Path, user: str = None):
+    """Schedule data scrapping on the basis of a feed configuration file"""
+    data_feed_cfg = load_toml_config(feed_cfg)
+    schedule = data_feed_cfg["data-feed"]["update_frequency"]
+    id = data_feed_cfg["data-feed"]["id"]
+    log_path = BERTREND_LOG_PATH if not user else BERTREND_LOG_PATH / "users" / user
+    log_path.mkdir(parents=True, exist_ok=True)
+    command = f"{sys.prefix}/bin/python -m bertrend_apps.data_provider scrape-feed {feed_cfg.resolve()} > {log_path}/cron_feed_{id}.log 2>&1"
+    add_job_to_crontab(schedule, command, "")
+
+
+def schedule_newsletter(
+    newsletter_cfg_path: Path,
+    data_feed_cfg_path: Path,
+    cuda_devices: str = BEST_CUDA_DEVICE,
+):
+    """Schedule data scrapping on the basis of a feed configuration file"""
+    newsletter_cfg = load_toml_config(newsletter_cfg_path)
+    schedule = newsletter_cfg["newsletter"]["update_frequency"]
+    id = newsletter_cfg["newsletter"]["id"]
+    command = f"{sys.prefix}/bin/python -m bertrend_apps.newsletters newsletters {newsletter_cfg_path.resolve()} {data_feed_cfg_path.resolve()} > {BERTREND_LOG_PATH}/cron_newsletter_{id}.log 2>&1"
+    env_vars = f"CUDA_VISIBLE_DEVICES={cuda_devices}"
+    add_job_to_crontab(schedule, command, env_vars)
+
+
+def check_if_scrapping_active_for_user(feed_id: str, user: str = None) -> bool:
+    """Checks if a given scrapping feed is active (registered in the crontab"""
+    if user:
+        return check_cron_job(rf"scrape-feed.*/users/{user}/{feed_id}_feed.toml")
+    else:
+        return check_cron_job(rf"scrape-feed.*/{feed_id}_feed.toml")
+
+
+def remove_scrapping_for_user(feed_id: str, user: str = None):
+    """Removes from the crontab the job matching the provided feed_id"""
+    if user:
+        return remove_from_crontab(rf"scrape-feed.*/users/{user}/{feed_id}_feed.toml")
+    else:
+        return remove_from_crontab(rf"scrape-feed.*/{feed_id}_feed.toml")

bertrend_apps/common/mail_utils.py

@@ -0,0 +1,91 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+
+import base64
+import os
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+from email.utils import COMMASPACE
+from pathlib import Path
+
+# Gmail API utils
+from google.auth.transport.requests import Request
+from google.oauth2.credentials import Credentials
+from google_auth_oauthlib.flow import InstalledAppFlow
+from googleapiclient.discovery import build
+from googleapiclient.errors import HttpError
+from loguru import logger
+
+from bertrend import BASE_PATH
+
+SCOPES = ["https://mail.google.com/"]  # full access to mail API
+FROM = "wattelse.ai@gmail.com"
+TOKEN_PATH = BASE_PATH / "gmail_token.json"
+DEFAULT_GMAIL_CREDENTIALS_PATH = (
+    Path(__file__).parent.parent / "config" / "gmail_credentials.json"
+)
+
+# Ensures to write with +rw for both user and groups
+os.umask(0o002)
+
+
+def get_credentials(
+    credentials_path: Path = DEFAULT_GMAIL_CREDENTIALS_PATH,
+) -> Credentials:
+    """Returns credentials for the user"""
+    creds = None
+    # The file token.json stores the user's access and refresh tokens, and is
+    # created automatically when the authorization flow completes for the first
+    # time.
+    logger.debug(f"Gmail credentials path: {credentials_path}")
+    logger.debug(f"Gmail token path: {TOKEN_PATH}")
+    if TOKEN_PATH.exists():
+        creds = Credentials.from_authorized_user_file(TOKEN_PATH, SCOPES)
+    # If there are no (valid) credentials available, let the user log in.
+    if not creds or not creds.valid:
+        if creds and creds.expired and creds.refresh_token:
+            creds.refresh(Request())
+        else:
+            flow = InstalledAppFlow.from_client_secrets_file(credentials_path, SCOPES)
+            creds = flow.run_local_server(port=0)
+        # Save the credentials for the next run
+        with open(TOKEN_PATH, "w") as token:
+            token.write(creds.to_json())
+    return creds
+
+
+def send_email(
+    credentials: Credentials,
+    subject: str,
+    recipients: list[str],
+    content: str,
+    content_type="html",
+):
+    try:
+        # Call the Gmail API
+        service = build("gmail", "v1", credentials=credentials)
+        message = MIMEMultipart()
+        message["To"] = COMMASPACE.join(recipients)
+        message["From"] = FROM
+        message["Subject"] = subject
+
+        # Record the MIME types of both parts - text/plain and text/html.
+        part1 = MIMEText(
+            content, "plain" if content_type in ["md", "text", "txt"] else content_type
+        )
+        message.attach(part1)
+
+        encoded_message = base64.urlsafe_b64encode(message.as_bytes()).decode()
+
+        create_message = {"raw": encoded_message}
+
+        # Send email using the Gmail API
+        email = (
+            service.users().messages().send(userId="me", body=create_message).execute()
+        )
+
+        logger.debug(f"Email sent: {email}")
+    except HttpError as error:
+        logger.error(f"An error occurred: {error}")

bertrend_apps/config/feeds/arxiv_feed.toml

@@ -0,0 +1,15 @@
+[data-feed]
+id = "feed_arxiv"
+query = "cat:cs.CL"
+provider = "arxiv"
+# maximum number of results *per day*
+max_results = 1000
+# number of days to retrieve (better if aligned with update-frequency if automated retrieval is activated)
+number_of_days = 7
+# output_directory is a relative path - base_data_dir_path/data_dir_path; the base_data_dir_path is controlled by the application
+feed_dir_path = "feed_arxiv"
+# language filter
+language = "en"
+# Frequency at which new data shall be gathered (in crontab format, cf https://crontab.guru/)
+# “At 22:00 on every day-of-week from Monday through Friday.”
+update_frequency = "0 1 * * 5"

bertrend_apps/config/feeds/decarbonation_feed.toml

@@ -0,0 +1,15 @@
+[data-feed]
+id = "feed_decarbonation"
+query = "décarbonation OR \"énergies renouvelables\" OR \"transition énergétique électrique\" OR \"smart grids\" OR \"stockage d'énergie\" OR \"efficacité énergétique\""
+provider = "google"
+# maximum number of results *per day*
+max_results = 50
+# number of days to retrieve (better if aligned with update-frequency if automated retrieval is activated)
+number_of_days = 7
+# output_directory is a relative path - base_data_dir_path/data_dir_path; the base_data_dir_path is controlled by the application
+feed_dir_path = "feed_decarbonation"
+# language filter
+language = "fr"
+# Frequency at which new data shall be gathered (in crontab format, cf https://crontab.guru/)
+# “At 22:00 on every day-of-week from Monday through Friday.”
+update_frequency = "0 2 * * 3"

bertrend_apps/config/feeds/llm_en_feed.toml

@@ -0,0 +1,15 @@
+[data-feed]
+id = "feed_llm_en"
+query = "LLM OR \"Retrieval Augmented Generation\""
+provider = "google"
+# maximum number of results *per day*
+max_results = 25
+# number of days to retrieve (better if aligned with update-frequency if automated retrieval is activated)
+number_of_days = 14
+# output_directory is a relative path - base_data_dir_path/data_dir_path; the base_data_dir_path is controlled by the application
+feed_dir_path = "feed_llm_en"
+# language filter
+language = "en"
+# Frequency at which new data shall be gathered (in crontab format, cf https://crontab.guru/)
+# “At 22:00 on every day-of-week from Monday through Friday.”
+update_frequency = "55 4 * * 4"

bertrend_apps/config/feeds/llm_feed.toml

@@ -0,0 +1,16 @@
+[data-feed]
+id = "feed_llm"
+# NB : LLM est aussi un mastère de droit !
+query = "LLM OR \"Retrieval Augmented Generation\""
+provider = "google"
+# maximum number of results *per day*
+max_results = 15
+# number of days to retrieve (better if aligned with update-frequency if automated retrieval is activated)
+number_of_days = 14
+# output_directory is a relative path - base_data_dir_path/data_dir_path; the base_data_dir_path is controlled by the application
+feed_dir_path = "feed_llm"
+# language filter
+language = "fr"
+# Frequency at which new data shall be gathered (in crontab format, cf https://crontab.guru/)
+# “At 22:00 on every day-of-week from Monday through Friday.”
+update_frequency = "0 5 * * 4"

bertrend_apps/config/feeds/nlp_feed.toml

@@ -0,0 +1,15 @@
+[data-feed]
+id = "feed_nlp"
+query = "NLP OR intelligence artificielle"
+provider = "google"
+# maximum number of results *per day*
+max_results = 50
+# number of days to retrieve (better if aligned with update-frequency if automated retrieval is activated)
+number_of_days = 7
+# output_directory is a relative path - base_data_dir_path/data_dir_path; the base_data_dir_path is controlled by the application
+feed_dir_path = "feed_nlp"
+# language filter
+language = "fr"
+# Frequency at which new data shall be gathered (in crontab format, cf https://crontab.guru/)
+# “At 22:00 on every day-of-week from Monday through Friday.”
+update_frequency = "0 3 * * 2"

bertrend_apps/config/feeds/rte_feed.toml

@@ -0,0 +1,15 @@
+[data-feed]
+id = "feed_rte"
+query = "électricité AND RTE"
+provider = "google"
+# maximum number of results *per day*
+max_results = 50
+# number of days to retrieve (better if aligned with update-frequency if automated retrieval is activated)
+number_of_days = 14
+# output_directory is a relative path - base_data_dir_path/data_dir_path; the base_data_dir_path is controlled by the application
+feed_dir_path = "feed_rte"
+# language filter
+language = "fr"
+# Frequency at which new data shall be gathered (in crontab format, cf https://crontab.guru/)
+# “At 22:00 on every day-of-week from Monday through Friday.”
+update_frequency = "0 4 * * 1"

bertrend_apps/config/gmail_credentials.json

@@ -0,0 +1 @@
+{"installed":{"client_id":"249845606098-tf75cv3rpsmcodl0k3um48s6gi9qlhbl.apps.googleusercontent.com","project_id":"wattelse","auth_uri":"https://accounts.google.com/o/oauth2/auth","token_uri":"https://oauth2.googleapis.com/token","auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs","client_secret":"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","redirect_uris":["http://localhost"]}}

bertrend_apps/config/newsletters/arxiv_newsletter_from_last.toml

@@ -0,0 +1,39 @@
+######## NEWSLETTER CONFIG ########
+[newsletter]
+id = "newsletter_arxiv"
+title = "Arxiv Newsletter"
+top_n_topics = 6
+top_n_docs = 4
+output_format = "html"
+# output_directory is a relative path - base_output_path/output_directory; the base_output_path is controlled by the application
+output_directory = "newsletter_arxiv"
+summarizer_class = "bertrend.services.summary.chatgpt_summarizer.GPTSummarizer"
+prompt_language = "en"
+# To automate the newsletters generation, e.g. “At 22:00 on every day-of-week from Monday through Friday.”
+update_frequency = "10 1 * * 5"
+recipients = "$NEWSLETTER_DEFAULT_RECIPIENTS"
+improve_topic_description = true
+# 'document' to summarize top n documents independently or 'topic' to generate a single topic summary
+summary_mode = "topic"
+#openai_model_name =
+
+######## LEARNING STRATEGY ########
+[learning_strategy]
+# Choose among:
+# - learn_from_scratch : uses all available data from feed to create the model
+# - learn_from_last: uses only the last feed data to create the model
+# - inference_only: do not retrain model; reuse existing bertopic model if available,
+#   otherwise, fallback to learn_from_scratch for the first run
+learning_strategy = "learn_from_last"
+split_data_by_paragraphs = "no"
+# model saving path (empty = do not save model)
+bertopic_model_path = "models/arxiv"
+
+######## TOPIC MODEL CONFIG ########
+[embedding_service]
+# local embedding model to be used
+model_name = "OrdalieTech/Solon-embeddings-base-0.1"
+
+[bertopic_parameters]
+language = "English"
+

bertrend_apps/config/newsletters/arxiv_newsletter_from_scratch.toml

@@ -0,0 +1,39 @@
+######## NEWSLETTER CONFIG ########
+[newsletter]
+id = "newsletter_arxiv"
+title = "Arxiv Newsletter"
+top_n_topics = 6
+top_n_docs = 4
+output_format = "html"
+# output_directory is a relative path - base_output_path/output_directory; the base_output_path is controlled by the application
+output_directory = "newsletter_arxiv"
+summarizer_class = "bertrend.services.summary.chatgpt_summarizer.GPTSummarizer"
+prompt_language = "en"
+# To automate the newsletters generation, e.g. “At 22:00 on every day-of-week from Monday through Friday.”
+update_frequency = "30 1 * * 5"
+recipients = "$NEWSLETTER_DEFAULT_RECIPIENTS"
+improve_topic_description = true
+# 'document' to summarize top n documents independently or 'topic' to generate a single topic summary
+summary_mode = "document"
+#openai_model_name =
+
+######## LEARNING STRATEGY ########
+[learning_strategy]
+# Choose among:
+# - learn_from_scratch : uses all available data from feed to create the model
+# - learn_from_last: uses only the last feed data to create the model
+# - inference_only: do not retrain model; reuse existing bertopic model if available,
+#   otherwise, fallback to learn_from_scratch for the first run
+learning_strategy = "learn_from_scratch"
+split_data_by_paragraphs = "no"
+# model saving path (empty = do not save model)
+bertopic_model_path = "models/arxiv"
+
+######## TOPIC MODEL CONFIG ########
+[embedding_service]
+# local embedding model to be used
+model_name = "OrdalieTech/Solon-embeddings-base-0.1"
+
+[bertopic_parameters]
+language = "English"
+

bertrend_apps/config/newsletters/decarbonation_newsletter_from_last.toml

@@ -0,0 +1,38 @@
+######## NEWSLETTER CONFIG ########
+[newsletter]
+id = "newsletter_decarbonation_from_last"
+title = "Newsletter Décarbonation"
+top_n_topics = 5
+top_n_docs = 5
+output_format = "html"
+# output_directory is a relative path - base_output_path/output_directory; the base_output_path is controlled by the application
+output_directory = "newsletter_rte"
+summarizer_class = "bertrend.services.summary.chatgpt_summarizer.GPTSummarizer"
+prompt_language = "fr"
+# To automate the newsletters generation, e.g. “At 22:00 on every day-of-week from Monday through Friday.”
+update_frequency = "15 2 * * 3"
+recipients = "$NEWSLETTER_DEFAULT_RECIPIENTS"
+improve_topic_description = true
+# 'document' to summarize top n documents independently or 'topic' to generate a single topic summary
+summary_mode = "topic"
+#openai_model_name =
+
+######## LEARNING STRATEGY ########
+[learning_strategy]
+# Choose among:
+# - learn_from_scratch : uses all available data from feed to create the model
+# - learn_from_last: uses only the last feed data to create the model
+# - inference_only: do not retrain model; reuse existing bertopic model if available,
+#   otherwise, fallback to learn_from_scratch for the first run
+learning_strategy = "learn_from_last"
+split_data_by_paragraphs = "yes"
+# model saving path (empty = do not save model)
+bertopic_model_path = "models/decarbonation_from_last"
+
+######## TOPIC MODEL CONFIG ########
+[embedding_service]
+# local embedding model to be used
+model_name = "OrdalieTech/Solon-embeddings-base-0.1"
+
+[bertopic_parameters]
+language = "French"

bertrend_apps/config/newsletters/decarbonation_newsletter_from_scratch.toml

@@ -0,0 +1,38 @@
+######## NEWSLETTER CONFIG ########
+[newsletter]
+id = "newsletter_decarbonation_from_scratch"
+title = "Newsletter Décarbonation"
+top_n_topics = 5
+top_n_docs = 5
+output_format = "html"
+# output_directory is a relative path - base_output_path/output_directory; the base_output_path is controlled by the application
+output_directory = "newsletter_rte"
+summarizer_class = "bertrend.services.summary.chatgpt_summarizer.GPTSummarizer"
+prompt_language = "fr"
+# To automate the newsletters generation, e.g. “At 22:00 on every day-of-week from Monday through Friday.”
+update_frequency = "30 2 * * 3"
+recipients = "$NEWSLETTER_DEFAULT_RECIPIENTS"
+improve_topic_description = true
+# 'document' to summarize top n documents independently or 'topic' to generate a single topic summary
+summary_mode = "document"
+#openai_model_name =
+
+######## LEARNING STRATEGY ########
+[learning_strategy]
+# Choose among:
+# - learn_from_scratch : uses all available data from feed to create the model
+# - learn_from_last: uses only the last feed data to create the model
+# - inference_only: do not retrain model; reuse existing bertopic model if available,
+#   otherwise, fallback to learn_from_scratch for the first run
+learning_strategy = "learn_from_scratch"
+split_data_by_paragraphs = "yes"
+# model saving path (empty = do not save model)
+bertopic_model_path = "models/decarbonation_from_scratch"
+
+######## TOPIC MODEL CONFIG ########
+[embedding_service]
+# local embedding model to be used
+model_name = "OrdalieTech/Solon-embeddings-base-0.1"
+
+[bertopic_parameters]
+language = "English"

bertrend_apps/config/newsletters/llm_en_newsletter_from_last.toml

@@ -0,0 +1,38 @@
+######## NEWSLETTER CONFIG ########
+[newsletter]
+id = "newsletter_LLM_en"
+title = "LLM Newsletter"
+top_n_topics = 6
+top_n_docs = 4
+output_format = "html"
+# output_directory is a relative path - base_output_path/output_directory; the base_output_path is controlled by the application
+output_directory = "newsletter_llm"
+summarizer_class = "bertrend.services.summary.chatgpt_summarizer.GPTSummarizer"
+prompt_language = "en"
+# To automate the newsletters generation, e.g. “At 22:00 on every day-of-week from Monday through Friday.”
+update_frequency = "30 5 * * 4"
+recipients = "$NEWSLETTER_DEFAULT_RECIPIENTS"
+improve_topic_description = true
+# 'document' to summarize top n documents independently or 'topic' to generate a single topic summary
+summary_mode = "topic"
+#openai_model_name =
+
+######## LEARNING STRATEGY ########
+[learning_strategy]
+# Choose among:
+# - learn_from_scratch : uses all available data from feed to create the model
+# - learn_from_last: uses only the last feed data to create the model
+# - inference_only: do not retrain model; reuse existing bertopic model if available,
+#   otherwise, fallback to learn_from_scratch for the first run
+learning_strategy = "learn_from_last"
+split_data_by_paragraphs = "yes"
+# model saving path (empty = do not save model)
+bertopic_model_path = "models/llm_from_last_en"
+
+######## TOPIC MODEL CONFIG ########
+[embedding_service]
+# local embedding model to be used
+model_name = "OrdalieTech/Solon-embeddings-base-0.1"
+
+[bertopic_parameters]
+language = "English"

bertrend_apps/config/newsletters/llm_newsletter_from_last.toml

@@ -0,0 +1,39 @@
+######## NEWSLETTER CONFIG ########
+[newsletter]
+id = "newsletter_LLM"
+title = "Newsletter LLM"
+top_n_topics = 5
+top_n_docs = 4
+output_format = "html"
+# output_directory is a relative path - base_output_path/output_directory; the base_output_path is controlled by the application
+output_directory = "newsletter_llm"
+summarizer_class = "bertrend.services.summary.chatgpt_summarizer.GPTSummarizer"
+prompt_language = "fr"
+# To automate the newsletters generation, e.g. “At 22:00 on every day-of-week from Monday through Friday.”
+update_frequency = "15 5 * * 4"
+recipients = "$NEWSLETTER_DEFAULT_RECIPIENTS"
+improve_topic_description = true
+# 'document' to summarize top n documents independently or 'topic' to generate a single topic summary
+summary_mode = "topic"
+#openai_model_name =
+
+######## LEARNING STRATEGY ########
+[learning_strategy]
+# Choose among:
+# - learn_from_scratch : uses all available data from feed to create the model
+# - learn_from_last: uses only the last feed data to create the model
+# - inference_only: do not retrain model; reuse existing bertopic model if available,
+#   otherwise, fallback to learn_from_scratch for the first run
+learning_strategy = "learn_from_last"
+split_data_by_paragraphs = "yes"
+# model saving path (empty = do not save model)
+bertopic_model_path = "models/llm_from_last"
+
+######## TOPIC MODEL CONFIG ########
+[embedding_service]
+# local embedding model to be used
+model_name = "OrdalieTech/Solon-embeddings-base-0.1"
+
+[bertopic_parameters]
+language = "French"
+

bertrend_apps/config/newsletters/nlp_newsletter_from_last.toml

@@ -0,0 +1,38 @@
+######## NEWSLETTER CONFIG ########
+[newsletter]
+id = "newsletter_NLP"
+title = "Newsletter NLP/IA"
+top_n_topics = 6
+top_n_docs = 5
+output_format = "html"
+# output_directory is a relative path - base_output_path/output_directory; the base_output_path is controlled by the application
+output_directory = "newsletter_nlp"
+summarizer_class = "bertrend.services.summary.chatgpt_summarizer.GPTSummarizer"
+prompt_language = "fr"
+# To automate the newsletters generation, e.g. “At 22:00 on every day-of-week from Monday through Friday.”
+update_frequency = "15 3 * * 2"
+recipients = "$NEWSLETTER_DEFAULT_RECIPIENTS"
+improve_topic_description = true
+# 'document' to summarize top n documents independently or 'topic' to generate a single topic summary
+summary_mode = "topic"
+#openai_model_name =
+
+######## LEARNING STRATEGY ########
+[learning_strategy]
+# Choose among:
+# - learn_from_scratch : uses all available data from feed to create the model
+# - learn_from_last: uses only the last feed data to create the model
+# - inference_only: do not retrain model; reuse existing bertopic model if available,
+#   otherwise, fallback to learn_from_scratch for the first run
+learning_strategy = "learn_from_last"
+split_data_by_paragraphs = "yes"
+# model saving path (empty = do not save model)
+bertopic_model_path = "models/nlp_from_last"
+
+######## TOPIC MODEL CONFIG ########
+[embedding_service]
+# local embedding model to be used
+model_name = "OrdalieTech/Solon-embeddings-base-0.1"
+
+[bertopic_parameters]
+language = "French"

bertrend_apps/config/newsletters/rte_newsletter_from_last.toml

@@ -0,0 +1,38 @@
+######## NEWSLETTER CONFIG ########
+[newsletter]
+id = "newsletter_rte_from_last"
+title = "Newsletter RTE"
+top_n_topics = 5
+top_n_docs = 4
+output_format = "html"
+# output_directory is a relative path - base_output_path/output_directory; the base_output_path is controlled by the application
+output_directory = "newsletter_rte"
+summarizer_class = "bertrend.services.summary.chatgpt_summarizer.GPTSummarizer"
+prompt_language = "fr"
+# To automate the newsletters generation, e.g. “At 22:00 on every day-of-week from Monday through Friday.”
+update_frequency = "15 4 * * 1"
+recipients = "$NEWSLETTER_DEFAULT_RECIPIENTS"
+improve_topic_description = true
+# 'document' to summarize top n documents independently or 'topic' to generate a single topic summary
+summary_mode = "topic"
+#openai_model_name =
+
+######## LEARNING STRATEGY ########
+[learning_strategy]
+# Choose among:
+# - learn_from_scratch : uses all available data from feed to create the model
+# - learn_from_last: uses only the last feed data to create the model
+# - inference_only: do not retrain model; reuse existing bertopic model if available,
+#   otherwise, fallback to learn_from_scratch for the first run
+learning_strategy = "learn_from_last"
+split_data_by_paragraphs = "yes"
+# model saving path (empty = do not save model)
+bertopic_model_path = "models/rte_from_last"
+
+######## TOPIC MODEL CONFIG ########
+[embedding_service]
+# local embedding model to be used
+model_name = "OrdalieTech/Solon-embeddings-base-0.1"
+
+[bertopic_parameters]
+language = "French"

bertrend_apps/config/newsletters/rte_newsletter_from_scratch.toml

@@ -0,0 +1,38 @@
+######## NEWSLETTER CONFIG ########
+[newsletter]
+id = "newsletter_rte_from_scratch"
+title = "Newsletter RTE"
+top_n_topics = 5
+top_n_docs = 4
+output_format = "html"
+# output_directory is a relative path - base_output_path/output_directory; the base_output_path is controlled by the application
+output_directory = "newsletter_rte"
+summarizer_class = "bertrend.services.summary.chatgpt_summarizer.GPTSummarizer"
+prompt_language = "fr"
+# To automate the newsletters generation, e.g. “At 22:00 on every day-of-week from Monday through Friday.”
+update_frequency = "30 4 * * 1"
+recipients = "$NEWSLETTER_DEFAULT_RECIPIENTS"
+improve_topic_description = true
+# 'document' to summarize top n documents independently or 'topic' to generate a single topic summary
+summary_mode = "document"
+#openai_model_name =
+
+######## LEARNING STRATEGY ########
+[learning_strategy]
+# Choose among:
+# - learn_from_scratch : uses all available data from feed to create the model
+# - learn_from_last: uses only the last feed data to create the model
+# - inference_only: do not retrain model; reuse existing bertopic model if available,
+#   otherwise, fallback to learn_from_scratch for the first run
+learning_strategy = "learn_from_scratch"
+split_data_by_paragraphs = "yes"
+# model saving path (empty = do not save model)
+bertopic_model_path = "models/rte_from_scratch"
+
+######## TOPIC MODEL CONFIG ########
+[embedding_service]
+# local embedding model to be used
+model_name = "OrdalieTech/Solon-embeddings-base-0.1"
+
+[bertopic_parameters]
+language = "French"