bertrend 0.3.12__tar.gz → 0.3.13__tar.gz

{bertrend-0.3.12 → bertrend-0.3.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: bertrend
-Version: 0.3.12
+Version: 0.3.13
 Summary: BERTrend analyses topic evolution over time using state-of-the-art transformer models: topic modelling and LLM
 License: LICENSE.md
 Keywords: topics,BERTopic,transformers,weak signals,LLM
@@ -59,6 +59,7 @@ Requires-Dist: sentencepiece (>=0.2.0)
 Requires-Dist: starlette (>=0.46.2)
 Requires-Dist: streamlit (>=1.46)
 Requires-Dist: streamlit-code-editor (>=0.1.22)
+Requires-Dist: streamlit-tags
 Requires-Dist: supervisor (>=4.2.5)
 Requires-Dist: thefuzz (>=0.22.1)
 Requires-Dist: tiktoken (>=0.8.0)

{bertrend-0.3.12 → bertrend-0.3.13}/bertrend/llm_utils/newsletter_model.py

@@ -2,8 +2,8 @@
 #  See AUTHORS.txt
 #  SPDX-License-Identifier: MPL-2.0
 #  This file is part of BERTrend.
+from typing import Optional
 
-from typing import List
 from pydantic import BaseModel
 from datetime import date
 
@@ -16,7 +16,7 @@ class Article(BaseModel):
     """Article model."""
 
     title: str
-    date: date
+    date: Optional[date]
     summary: str | None = None
     source: str | None = None
     url: str | None = None
@@ -26,9 +26,9 @@ class Topic(BaseModel):
     """Topic model."""
 
     title: str
-    hashtags: List[str]
+    hashtags: list[str] | None = None
     summary: str | None = None
-    articles: List[Article]
+    articles: list[Article]
     topic_type: str = (
         STRONG_TOPIC_TYPE  # Default value "strong", can be "weak" or "noise"
     )
@@ -38,7 +38,8 @@ class Newsletter(BaseModel):
     """Newsletter model."""
 
     title: str
-    period_start_date: date
-    period_end_date: date
-    topics: List[Topic]
+    reference_period: date = None  # has the precedence over the two other dates
+    period_start_date: date = None
+    period_end_date: date = None
+    topics: list[Topic]
     debug_info: dict | None = None

{bertrend-0.3.12 → bertrend-0.3.13}/bertrend/llm_utils/newsletter_outlook_template.html

@@ -58,13 +58,21 @@
                     <tr>
                         <td>
                             <h1 style="font-size: 32px; color: #333; margin: 0 0 10px 0; font-family: Arial, sans-serif;">{{ newsletter.title }}</h1>
-                            <p style="font-size: 18px; color: #555; margin: 0 0 30px 0; font-family: Arial, sans-serif;">
-                                {% if language == 'en' %}
-                                    Period: {{ newsletter.period_start_date }} to {{ newsletter.period_end_date }}
-                                {% else %}
-                                    Période : {{ newsletter.period_start_date }} au {{ newsletter.period_end_date }}
-                                {% endif %}
-                            </p>
+                                <p style="font-size: 18px; color: #555; margin: 0 0 30px 0; font-family: Arial, sans-serif;">
+                                    {% if newsletter.reference_period %}
+                                        {% if language == 'en' %}
+                                            Period: {{ newsletter.reference_period }}
+                                        {% else %}
+                                            Période : {{ newsletter.reference_period }}
+                                        {% endif %}
+                                    {% elif newsletter.period_start_date and newsletter.period_end_date %}
+                                        {% if language == 'en' %}
+                                            Period: {{ newsletter.period_start_date }} to {{ newsletter.period_end_date }}
+                                        {% else %}
+                                            Période : {{ newsletter.period_start_date }} au {{ newsletter.period_end_date }}
+                                        {% endif %}
+                                    {% endif %}
+                                </p>
                         </td>
                     </tr>
                 </table>
@@ -123,7 +131,9 @@
 
                                                     <!-- Date and Source -->
                                                     <div style="font-size: 10px; color: #666; margin-bottom: 8px; font-family: Arial, sans-serif;">
+                                                        {% if article.date %}
                                                         <span style="font-weight: bold;">{{ article.date }}</span>
+                                                        {% endif %}
                                                         {% if article.source %}
                                                         <span style="margin: 0 5px;">|</span>
                                                         <span style="font-style: italic;">
@@ -150,6 +160,10 @@
                                             </tr>
                                         </table>
                                         {% endfor %}
+
+                                        {% block detailed_topic_analysis scoped %}
+                                            <!-- Placeholder for detailed analysis -->
+                                        {% endblock %}
                                     </td>
                                 </tr>
                             </table>
@@ -174,6 +188,8 @@
             </td>
         </tr>
     </table>
-</body>
 
-</html>
+    </body>
+
+</html>
+

{bertrend-0.3.12 → bertrend-0.3.13}/bertrend/llm_utils/newsletter_template.html

@@ -27,10 +27,18 @@
         <header>
             <h1>{{ newsletter.title }}</h1>
             <p class="period">
-                {% if language == 'en' %}
-                    Period: {{ newsletter.period_start_date }} to {{ newsletter.period_end_date }}
-                {% else %}
-                    Période : {{ newsletter.period_start_date }} au {{ newsletter.period_end_date }}
+                {% if newsletter.reference_period %}
+                    {% if language == 'en' %}
+                        Period: {{ newsletter.reference_period }}
+                    {% else %}
+                        Période : {{ newsletter.reference_period }}
+                    {% endif %}
+                {% elif newsletter.period_start_date and newsletter.period_end_date %}
+                    {% if language == 'en' %}
+                        Period: {{ newsletter.period_start_date }} to {{ newsletter.period_end_date }}
+                    {% else %}
+                        Période : {{ newsletter.period_start_date }} au {{ newsletter.period_end_date }}
+                    {% endif %}
                 {% endif %}
             </p>
         </header>
@@ -61,7 +69,9 @@
                         {% endif %}
                         <!-- Date and Source Inline -->
                         <div class="timestamp">
-                            <div class="date">{{ article.date }}</div>
+                            {% if article.date %}
+                                <div class="date">{{ article.date }}</div>
+                            {% endif %}
                             {% if article.source %}
                                 <span class="separator">|</span>
                                 <div class="source">
@@ -87,6 +97,11 @@
                     </div>
                     {% endfor %}
                 </div>
+
+                <!-- Placeholder for additional info -->
+                {% block detailed_topic_analysis with topic=topic %}
+                {% endblock %}
+
             </section>
             {% endfor %}
         </main>

{bertrend-0.3.12 → bertrend-0.3.13}/bertrend/tests/test_newsletter_model.py

@@ -127,11 +127,6 @@ class TestTopic:
             Topic(hashtags=["#test"], articles=articles)
         assert "title" in str(exc_info.value)
 
-        # Missing hashtags
-        with pytest.raises(ValidationError) as exc_info:
-            Topic(title="Test Topic", articles=articles)
-        assert "hashtags" in str(exc_info.value)
-
         # Missing articles
         with pytest.raises(ValidationError) as exc_info:
             Topic(title="Test Topic", hashtags=["#test"])
@@ -246,30 +241,13 @@ class TestNewsletter:
         # Missing title
         with pytest.raises(ValidationError) as exc_info:
             Newsletter(
+                reference_period=date(2024, 1, 31),
                 period_start_date=date(2024, 1, 1),
                 period_end_date=date(2024, 1, 31),
                 topics=topics,
             )
         assert "title" in str(exc_info.value)
 
-        # Missing period_start_date
-        with pytest.raises(ValidationError) as exc_info:
-            Newsletter(
-                title="Test Newsletter",
-                period_end_date=date(2024, 1, 31),
-                topics=topics,
-            )
-        assert "period_start_date" in str(exc_info.value)
-
-        # Missing period_end_date
-        with pytest.raises(ValidationError) as exc_info:
-            Newsletter(
-                title="Test Newsletter",
-                period_start_date=date(2024, 1, 1),
-                topics=topics,
-            )
-        assert "period_end_date" in str(exc_info.value)
-
         # Missing topics
         with pytest.raises(ValidationError) as exc_info:
             Newsletter(
@@ -399,6 +377,7 @@ class TestIntegration:
             title="Test Newsletter",
             period_start_date=date(2024, 1, 1),
             period_end_date=date(2024, 1, 31),
+            reference_period=date(2024, 1, 31),
             topics=[topic],
         )
 
@@ -409,6 +388,7 @@ class TestIntegration:
         assert data["title"] == "Test Newsletter"
         assert data["period_start_date"] == date(2024, 1, 1)
         assert data["period_end_date"] == date(2024, 1, 31)
+        assert data["reference_period"] == date(2024, 1, 31)
         assert len(data["topics"]) == 1
         assert data["topics"][0]["title"] == "Test Topic"
         assert len(data["topics"][0]["articles"]) == 1
@@ -420,4 +400,5 @@ class TestIntegration:
         assert newsletter_from_dict.title == newsletter.title
         assert newsletter_from_dict.period_start_date == newsletter.period_start_date
         assert newsletter_from_dict.period_end_date == newsletter.period_end_date
+        assert newsletter_from_dict.reference_period == newsletter.reference_period
         assert len(newsletter_from_dict.topics) == len(newsletter.topics)

bertrend-0.3.13/bertrend_apps/common/mail_utils.py

@@ -0,0 +1,131 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+
+import base64
+import os
+from email.message import EmailMessage
+from email.utils import COMMASPACE
+import mimetypes
+from pathlib import Path
+
+# Gmail API utils
+from google.auth.transport.requests import Request
+from google.oauth2.credentials import Credentials
+from google_auth_oauthlib.flow import InstalledAppFlow
+from googleapiclient.discovery import build
+from googleapiclient.errors import HttpError
+from loguru import logger
+
+from bertrend import BASE_PATH
+
+SCOPES = ["https://mail.google.com/"]  # full access to mail API
+FROM = "wattelse.ai@gmail.com"
+TOKEN_PATH = BASE_PATH / "gmail_token.json"
+DEFAULT_GMAIL_CREDENTIALS_PATH = (
+    Path(__file__).parent.parent / "config" / "gmail_credentials.json"
+)
+
+# Ensures to write with +rw for both user and groups
+os.umask(0o002)
+
+
+def get_credentials(
+    credentials_path: Path = DEFAULT_GMAIL_CREDENTIALS_PATH,
+) -> Credentials:
+    """Returns credentials for the user"""
+    creds = None
+    # The file token.json stores the user's access and refresh tokens, and is
+    # created automatically when the authorization flow completes for the first
+    # time.
+    logger.debug(f"Gmail credentials path: {credentials_path}")
+    logger.debug(f"Gmail token path: {TOKEN_PATH}")
+    if TOKEN_PATH.exists():
+        creds = Credentials.from_authorized_user_file(TOKEN_PATH, SCOPES)
+    # If there are no (valid) credentials available, let the user log in.
+    if not creds or not creds.valid:
+        if creds and creds.expired and creds.refresh_token:
+            creds.refresh(Request())
+        else:
+            flow = InstalledAppFlow.from_client_secrets_file(credentials_path, SCOPES)
+            creds = flow.run_local_server(port=0)
+        # Save the credentials for the next run
+        with open(TOKEN_PATH, "w") as token:
+            token.write(creds.to_json())
+    return creds
+
+
+def send_email(
+    credentials,
+    subject: str,
+    recipients: list[str],
+    content: str | Path,
+    content_type: str = "html",
+    file_name: str = None,
+) -> None:
+    """
+    Send an e-mail with Gmail API.
+
+    If *content* is a valid path it is attached as a file (with an optional filename),
+    otherwise it is used as the message body.
+    """
+    try:
+        service = build("gmail", "v1", credentials=credentials)
+
+        msg = EmailMessage()
+        msg["Subject"] = subject
+        msg["From"] = FROM
+        msg["To"] = COMMASPACE.join(recipients)
+
+        # ------------------------------------------------------------------ #
+        # 1. Attach a file if *content* points to an existing file            #
+        # ------------------------------------------------------------------ #
+        if isinstance(content, (str, Path)) and Path(content).is_file():
+            file_path = Path(content)
+
+            # Body text so the message is not “empty”
+            msg.set_content(
+                f"Please find {file_path.name} attached.",
+                subtype="plain",
+            )
+
+            mime_type, _ = mimetypes.guess_type(file_path)
+            mime_type = mime_type or "application/octet-stream"
+            maintype, subtype = mime_type.split("/", 1)
+
+            with file_path.open("rb") as fp:
+                msg.add_attachment(
+                    fp.read(),
+                    maintype=maintype,
+                    subtype=subtype,
+                    filename=file_path.name if not file_name else file_name,
+                )
+
+        # ------------------------------------------------------------------ #
+        # 2. Otherwise use *content* as the body                             #
+        # ------------------------------------------------------------------ #
+        else:
+            subtype = (
+                "plain"
+                if content_type.lower() in {"md", "text", "txt"}
+                else content_type
+            )
+            msg.set_content(content, subtype=subtype)
+
+        # ------------------------------------------------------------------ #
+        # 3. Send through Gmail API                                          #
+        # ------------------------------------------------------------------ #
+        raw_message = base64.urlsafe_b64encode(msg.as_bytes()).decode()
+        service.users().messages().send(
+            userId="me", body={"raw": raw_message}
+        ).execute()
+
+        logger.debug("E-mail successfully sent.")
+
+    except HttpError as err:
+        logger.error("Gmail API error: %s", err)
+    except FileNotFoundError:
+        logger.error("File not found: %s", content)
+    except Exception as err:
+        logger.exception("Unexpected error: %s", err)

{bertrend-0.3.12 → bertrend-0.3.13}/bertrend_apps/prospective_demo/__init__.py

@@ -34,6 +34,7 @@ DEFAULT_ANALYSIS_CFG = {
         "granularity": DEFAULT_GRANULARITY,
         "window_size": DEFAULT_WINDOW_SIZE,
         "language": "French",
+        "split_by_paragraph": True,
     },
     "analysis_config": {
         "topic_evolution": True,

{bertrend-0.3.12 → bertrend-0.3.13}/bertrend_apps/prospective_demo/app.py

@@ -2,6 +2,7 @@
 #  See AUTHORS.txt
 #  SPDX-License-Identifier: MPL-2.0
 #  This file is part of BERTrend.
+from importlib.metadata import version
 
 import torch
 
@@ -43,6 +44,29 @@ AUTHENTIFICATION = True
 # AUTHENTIFICATION = False
 
 
+def display_version_number():
+    # Display version at the bottom (using st.markdown and CSS for fixed footer)
+    footer = f"""
+    <style>
+    .footer {{
+        position: fixed;
+        right: 10px;
+        bottom: 10px;
+        color: #666;
+        font-size: 0.8em;
+        opacity: 0.6;
+        user-select: none;
+        pointer-events: none;
+        background: none;
+    }}
+    </style>
+    <div class="footer">
+        Version: {version('bertrend')}
+    </div>
+    """
+    st.markdown(footer, unsafe_allow_html=True)
+
+
 def main():
     """Main page"""
     set_internationalization_language("fr")
@@ -54,6 +78,8 @@ def main():
         page_icon=":part_alternation_mark:",
     )
 
+    display_version_number()
+
     st.title(":part_alternation_mark: " + page_title)
 
     if AUTHENTIFICATION:

bertrend-0.3.13/bertrend_apps/prospective_demo/data_model.py

@@ -0,0 +1,19 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+from bertrend.llm_utils.newsletter_model import Topic, Newsletter
+from bertrend.trend_analysis.data_structure import SignalAnalysis, TopicSummaryList
+
+
+class TopicOverTime(Topic):
+    """Topic model."""
+
+    topic_evolution: TopicSummaryList | None = None
+    topic_analysis: SignalAnalysis | None = None
+
+
+class DetailedNewsletter(Newsletter):
+    """Newsletter model with detailed information about topic evolution over time."""
+
+    topics: list[TopicOverTime]