bertrend 0.4.0__tar.gz → 0.4.1__tar.gz

{bertrend-0.4.0 → bertrend-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bertrend
-Version: 0.4.0
+Version: 0.4.1
 Summary: BERTrend analyses topic evolution over time using state-of-the-art transformer models: topic modelling and LLM
 License: LICENSE.md
 License-File: AUTHORS.txt
@@ -28,7 +28,9 @@ Requires-Dist: dateparser (>=1.2.0)
 Requires-Dist: dill (>=0.4.0)
 Requires-Dist: fastapi (>=0.120.0)
 Requires-Dist: feedparser (>=6.0.11)
+Requires-Dist: folium (>=0.20.0)
 Requires-Dist: gensim (>=4.3.3)
+Requires-Dist: geopy (>=2.4.1)
 Requires-Dist: google-api-python-client (>=2.148.0)
 Requires-Dist: google-auth-oauthlib (>=1.2.1)
 Requires-Dist: googlenewsdecoder (>=0.1.7)
@@ -69,6 +71,7 @@ Requires-Dist: sentencepiece (>=0.2.0)
 Requires-Dist: starlette (>=0.46.2)
 Requires-Dist: streamlit (>=1.51.0)
 Requires-Dist: streamlit-code-editor (>=0.1.22)
+Requires-Dist: streamlit-folium (>=0.26.1)
 Requires-Dist: supervisor (>=4.3.0)
 Requires-Dist: thefuzz (>=0.22.1)
 Requires-Dist: tiktoken (>=0.8.0)

bertrend-0.4.1/bertrend/bertrend_apps/exploration/geoviz/.streamlit/config.toml

@@ -0,0 +1,6 @@
+[theme]
+primaryColor = "royalblue"
+baseFontSize = 13
+
+[server]
+port = 8089

bertrend-0.4.1/bertrend/bertrend_apps/exploration/geoviz/app.py

@@ -0,0 +1,174 @@
+#  Copyright (c) 2024-2026, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+
+import backend  # Importing our isolated logic
+import folium
+import pandas as pd
+import streamlit as st
+from dotenv import load_dotenv
+from folium.plugins import MarkerCluster
+from streamlit_folium import st_folium
+
+# Load environment variables at module import
+load_dotenv(override=True)
+
+# --- Page Config ---
+st.set_page_config(page_title="GeoNews Agent", layout="wide", page_icon="🌍")
+
+st.title("🌍 GeoNews Mapper")
+st.markdown("""
+    Extract geographical insights from news articles using AI. 
+    This tool detects cities, departments, and regions and maps them.
+""")
+
+# --- Sidebar: Configuration ---
+with st.sidebar:
+    st.header("Settings")
+
+    st.markdown("---")
+    st.markdown("**Hierarchy Logic:**")
+    st.caption("1. **Ville** (City)")
+    st.caption("2. **Département**")
+    st.caption("3. **Région**")
+
+# --- Main Input Section ---
+tab1, tab2 = st.tabs(["🔗 Paste URLs", "📂 Upload Data"])
+
+input_data = []
+input_type = ""
+
+examples = [
+    "https://www.francebleu.fr/provence-alpes-cote-d-azur/bouches-du-rhone-13/ligne-tht-en-camargue-malgre-les-risques-pour-les-especes-protegees-rte-maintient-son-projet-5901415",
+    "https://maritima.fr/actualites/economie/fos-sur-mer/10613/ligne-tres-haute-tension-vers-fos-letat-veut-que-rte-poursuive-les-travaux",
+]
+
+with tab1:
+    urls_input = st.text_area(
+        "Enter News URLs (one per line)",
+        value="\n".join(examples),
+        height=150,
+        placeholder="https://www.lemonde.fr/...\nhttps://www.lefigaro.fr/...",
+    )
+    if urls_input:
+        input_data = [url.strip() for url in urls_input.split("\n") if url.strip()]
+        input_type = "url"
+
+with tab2:
+    uploaded_file = st.file_uploader(
+        "Upload CSV or Excel (Column must be named 'text' or 'url')",
+        type=["csv", "xlsx"],
+    )
+    if uploaded_file:
+        if uploaded_file.name.endswith(".csv"):
+            df_upload = pd.read_csv(uploaded_file)
+        else:
+            df_upload = pd.read_excel(uploaded_file)
+
+        # Detect column
+        if "url" in df_upload.columns:
+            input_data = df_upload["url"].tolist()
+            input_type = "url"
+        elif "text" in df_upload.columns:
+            input_data = df_upload["text"].tolist()
+            input_type = "text"
+        else:
+            st.error("File must contain a 'url' or 'text' column.")
+
+# --- Execution ---
+if st.button("🚀 Extract & Map Locations"):
+    if not input_data:
+        st.warning("Please provide input URLs or text.")
+    else:
+        with st.spinner("🕵️‍♂️ Agent is reading articles and extracting locations..."):
+            # Call the backend
+            result_df = backend.extract_geo_data(input_data, input_type)
+
+            if result_df.empty:
+                st.error("No locations found or extraction failed.")
+            else:
+                st.session_state["geo_data"] = result_df
+                st.success(f"Extracted {len(result_df)} locations!")
+
+# --- Visualization Section ---
+if "geo_data" in st.session_state:
+    df = st.session_state["geo_data"]
+
+    st.divider()
+
+    # 1. Search / Filters
+    col_filter1, col_filter2 = st.columns(2)
+    with col_filter1:
+        # Get unique regions, filter out Nones
+        regions = ["All"] + sorted([x for x in df["region"].unique() if x])
+        selected_region = st.selectbox("Filter by Region", regions)
+
+    with col_filter2:
+        search_query = st.text_input("Search for a specific City or Department")
+
+    # Apply filters
+    filtered_df = df.copy()
+    if selected_region != "All":
+        filtered_df = filtered_df[filtered_df["region"] == selected_region]
+
+    if search_query:
+        filtered_df = filtered_df[
+            filtered_df["city"].str.contains(search_query, case=False, na=False)
+            | filtered_df["department"].str.contains(search_query, case=False, na=False)
+        ]
+
+    # 2. Map Layout
+    col_map, col_data = st.columns([2, 1])
+
+    with col_map:
+        st.subheader("Interactive Map")
+
+        # Initialize Map centered on France
+        m = folium.Map(location=[46.603354, 1.888334], zoom_start=6)
+
+        # Marker Cluster Group (Prevents overwhelm)
+        marker_cluster = MarkerCluster().add_to(m)
+
+        for idx, row in filtered_df.iterrows():
+            # Determine what to show as the title
+            if row["specific_location"]:
+                display_title = row["specific_location"]
+                display_subtitle = f"{row['city']}, {row['country']}"
+            else:
+                display_title = row["city"]
+                display_subtitle = f"{row['department'] or ''} - {row['country']}"
+
+            html = f"""
+            <b>{display_title}</b><br>
+            <span style='color:grey; font-size:10px'>{display_subtitle}</span><br>
+            <hr style='margin:5px 0'>
+            <i>"{row["snippet"]}"</i><br>
+            <a href='{row["source"]}' target='_blank'>Source</a>
+            """
+
+            folium.Marker(
+                location=[row["lat"], row["lon"]],
+                tooltip=row["city"],
+                popup=folium.Popup(html, max_width=300),
+                icon=folium.Icon(color="blue", icon="info-sign"),
+            ).add_to(marker_cluster)
+
+        st_folium(m, width="100%", height=600)
+
+    with col_data:
+        st.subheader("Extracted Data")
+        st.dataframe(
+            filtered_df[["city", "department", "region", "snippet"]],
+            hide_index=True,
+            height=600,
+        )
+
+    # Export
+    csv = filtered_df.to_csv(index=False).encode("utf-8")
+    st.download_button(
+        "📥 Download Data as CSV",
+        data=csv,
+        file_name="geo_news_data.csv",
+        mime="text/csv",
+    )

bertrend-0.4.1/bertrend/bertrend_apps/exploration/geoviz/backend.py

@@ -0,0 +1,190 @@
+from typing import List, Optional
+
+import pandas as pd
+from geopy.extra.rate_limiter import RateLimiter
+from geopy.geocoders import Nominatim
+from goose3 import Goose
+from pydantic import BaseModel, Field
+
+from bertrend.llm_utils.openai_client import OpenAI_Client
+
+
+# --- Data Models ---
+class GeoLocation(BaseModel):
+    specific_location: Optional[str] = Field(
+        None,
+        description="A specific landmark, building, district, or street (e.g., 'Tour Eiffel', 'Gare de Lyon', 'Brooklyn Bridge').",
+    )
+    city: str = Field(..., description="The city or municipality.")
+    department: Optional[str] = Field(
+        None,
+        description="The administrative department, state, or province (e.g., 'Gironde', 'California').",
+    )
+    region: Optional[str] = Field(
+        None,
+        description="The larger administrative region (e.g., 'Nouvelle-Aquitaine').",
+    )
+    country: str = Field(
+        ...,
+        description="The country where this location is found (e.g., 'France', 'USA').",
+    )
+    context_snippet: str = Field(
+        ...,
+        description="A short snippet of text justifying why this location was extracted.",
+    )
+
+
+class ArticleLocations(BaseModel):
+    locations: List[GeoLocation]
+
+
+# --- Helper Functions ---
+
+
+def fetch_text_from_url(url: str) -> str:
+    """
+    Uses Goose3 to extract the main content of the article,
+    stripping out ads, menus, and html clutter.
+    """
+    try:
+        g = Goose()
+        article = g.extract(url=url)
+        text = article.cleaned_text
+        g.close()
+
+        if not text:
+            return ""
+
+        # Truncate if extremely long to save OpenAI tokens
+        return text[:15000]
+
+    except Exception as e:
+        print(f"Goose extraction error for {url}: {e}")
+        return ""
+
+
+def geocode_locations(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Adds Lat/Lon using a hierarchical strategy to ensure the most precise
+    location is found while maintaining a fallback for broader searches.
+    """
+    # Unique user_agent prevents 403 errors
+    geolocator = Nominatim(user_agent="geo_news_agent_app_v5")
+    geocode = RateLimiter(geolocator.geocode, min_delay_seconds=1)
+
+    lats = []
+    lons = []
+
+    print("Geocoding locations...")
+
+    for _, row in df.iterrows():
+        location = None
+
+        # 1. Strategy: Specific Landmark Precision
+        # Query: "Tour Eiffel, Paris, France"
+        if row.get("specific_location"):
+            query = f"{row['specific_location']}, {row['city']}, {row['country']}"
+            try:
+                location = geocode(query)
+            except:
+                pass
+
+        # 2. Strategy: Administrative Precision
+        # Query: "Paris, Ile-de-France, France"
+        if not location and row.get("department"):
+            query = f"{row['city']}, {row['department']}, {row['country']}"
+            try:
+                location = geocode(query)
+            except:
+                pass
+
+        # 3. Strategy: City Fallback
+        # Query: "Paris, France"
+        if not location:
+            query = f"{row['city']}, {row['country']}"
+            try:
+                location = geocode(query)
+            except:
+                pass
+
+        if location:
+            lats.append(location.latitude)
+            lons.append(location.longitude)
+        else:
+            lats.append(None)
+            lons.append(None)
+
+    df["lat"] = lats
+    df["lon"] = lons
+
+    # Return only rows where we successfully found coordinates
+    return df.dropna(subset=["lat", "lon"])
+
+
+# --- Main Agent Function ---
+
+
+def extract_geo_data(inputs: List[str], input_type: str) -> pd.DataFrame:
+    """
+    Main pipeline:
+    1. Gets text (using Goose3 if URL).
+    2. Calls OpenAI to extract structured hierarchy including specific landmarks and country.
+    3. Aggregates results and geocodes them.
+    """
+    client = OpenAI_Client()
+    all_results = []
+
+    for item in inputs:
+        # 1. Prepare Text
+        if input_type == "url":
+            text_content = fetch_text_from_url(item)
+            source_id = item
+        else:
+            text_content = item
+            source_id = "Uploaded Text"
+
+        # Skip empty extractions
+        if not text_content or len(text_content) < 50:
+            continue
+
+        # 2. Call OpenAI Agent
+        try:
+            system_prompt = (
+                "You are a geographical extraction specialist. Extract all relevant physical locations mentioned in the text. "
+                "For each location, provide the following hierarchy:\n"
+                "1. **Specific Location**: A landmark, building, square, or district (e.g., 'Louvre Museum', 'La Défense'). If none, leave null.\n"
+                "2. **City**: The municipality.\n"
+                "3. **Department**: The state, province, or department.\n"
+                "4. **Country**: Crucial. Infer it if not explicit (e.g., London -> UK).\n"
+                "Provide a context snippet explaining the relevance."
+            )
+
+            extracted = client.parse(
+                system_prompt=system_prompt,
+                user_prompt=text_content,
+                response_format=ArticleLocations,
+            )
+
+            # 3. Flatten results
+            for loc in extracted.locations:
+                all_results.append(
+                    {
+                        "source": source_id,
+                        "specific_location": loc.specific_location,
+                        "city": loc.city,
+                        "department": loc.department,
+                        "region": loc.region,
+                        "country": loc.country,
+                        "snippet": loc.context_snippet,
+                    }
+                )
+
+        except Exception as e:
+            print(f"Error processing {source_id}: {e}")
+
+    df = pd.DataFrame(all_results)
+
+    if not df.empty:
+        df = geocode_locations(df)
+
+    return df

{bertrend-0.4.0 → bertrend-0.4.1}/bertrend/config/services_default_config.toml

@@ -29,4 +29,8 @@ base_url = "$OPENAI_BASE_URL"
 model = "$OPENAI_DEFAULT_MODEL"
 temperature = 0.1
 max_output_tokens = 2048
-system_prompt = "You are a helpful assistant, skilled in detailing topic evolution over time for the detection of emerging trends and signals."
+system_prompt = "You are a helpful assistant, skilled in detailing topic evolution over time for the detection of emerging trends and signals."
+
+# Summary service parameters
+[summary_service]
+url = "$SUMMARY_SERVICE_URL"

{bertrend-0.4.0 → bertrend-0.4.1}/bertrend/services/scheduling/docker-compose.yml

@@ -9,18 +9,17 @@ services:
       - "8882:8000"
     volumes:
       # Mount data directory for persistent SQLite database
-      - $HOME/.bertrend/db:$HOME/.bertrend/db
+      # Host path uses $HOME, container path uses /home/user
+      - ${HOME}/.bertrend/db:/home/user/.bertrend/db
     environment:
       - NO_PROXY=$NO_PROXY # to avoid communication issues
       - no_proxy=$no_proxy # to avoid communication issues
       # Application settings
       - TZ=Europe/Paris
       - PYTHONUNBUFFERED=1
-      # Optional: Log level
       - LOG_LEVEL=DEBUG
-      # Optional: Number of worker processes
       - MAX_WORKERS=100
-      - HOME=$HOME
+      - HOME=/home/user  # Generic home path in container
     restart: always
     logging:
       driver: "json-file"
@@ -35,7 +34,6 @@ services:
       start_period: 10s
     networks:
       - scheduler-network
-    # Resource limits (adjust based on your needs)
     deploy:
       resources:
         limits:

bertrend-0.4.1/bertrend/services/scheduling/start_docker_scheduling_service.sh

@@ -0,0 +1,16 @@
+#
+# Copyright (c) 2024-2026, RTE (https://www.rte-france.com)
+# See AUTHORS.txt
+# SPDX-License-Identifier: MPL-2.0
+# This file is part of BERTrend.
+#
+
+# Create directory on host
+mkdir -p ~/.bertrend/db
+
+# Export your UID/GID (add to your shell profile for persistence)
+export UID=$(id -u)
+export GID=$(id -g)
+
+# Run docker compose
+docker compose up --build --force-recreate -d

{bertrend-0.4.0 → bertrend-0.4.1}/bertrend/services/summarizer.py

@@ -34,7 +34,7 @@ class Summarizer(ABC):
         """Basic implementation of batch summarization. Can be overridden by subclasses."""
         return [
             self.generate_summary(
-                article_text=t,
+                t,
                 max_sentences=max_sentences,
                 max_words=max_words,
                 max_length_ratio=max_length_ratio,

bertrend-0.4.1/bertrend/services/summary_client.py

@@ -0,0 +1,70 @@
+#  Copyright (c) 2024-2026, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+
+import requests
+
+
+class SummaryAPIClient:
+    """Thin HTTP client for the BERTrend Summarization Service."""
+
+    def __init__(self, url: str = "http://localhost:6465"):
+        self.url = url.rstrip("/")
+
+    def health(self) -> dict:
+        """Check service health."""
+        resp = requests.get(f"{self.url}/health")
+        resp.raise_for_status()
+        return resp.json()
+
+    def list_summarizers(self) -> list[dict]:
+        """List available summarizers and their status."""
+        resp = requests.get(f"{self.url}/summarizers")
+        resp.raise_for_status()
+        return resp.json()
+
+    def summarize(
+        self,
+        text: str | list[str],
+        summarizer_type: str = "llm",
+        language: str = "fr",
+        max_sentences: int = 3,
+        max_words: int | None = None,
+        max_length_ratio: float = 0.2,
+    ) -> dict:
+        """Summarize one or more texts.
+
+        Parameters
+        ----------
+        text : str | list[str]
+            Text(s) to summarize.
+        summarizer_type : str
+            Summarizer backend: "llm", "extractive", or "enhanced".
+        language : str
+            Prompt language ("fr" or "en").
+        max_sentences : int
+            Maximum number of sentences in the summary.
+        max_words : int | None
+            Maximum number of words (optional).
+        max_length_ratio : float
+            Maximum ratio of summary length to original.
+
+        Returns
+        -------
+        dict
+            Response with keys: summaries, summarizer_type, language, processing_time_ms.
+        """
+        payload = {
+            "text": text,
+            "summarizer_type": summarizer_type,
+            "language": language,
+            "max_sentences": max_sentences,
+            "max_length_ratio": max_length_ratio,
+        }
+        if max_words is not None:
+            payload["max_words"] = max_words
+
+        resp = requests.post(f"{self.url}/summarize", json=payload)
+        resp.raise_for_status()
+        return resp.json()

bertrend-0.4.1/bertrend/services/summary_server/config/default_config.toml

@@ -0,0 +1,3 @@
+host = "0.0.0.0"
+port = 8885
+number_workers = 1

bertrend-0.4.1/bertrend/services/summary_server/config/settings.py

@@ -0,0 +1,32 @@
+#  Copyright (c) 2024-2026, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+
+import os
+import tomllib
+from pathlib import Path
+
+from pydantic_settings import BaseSettings
+
+DEFAULT_CONFIG_FILE = Path(__file__).parent / "default_config.toml"
+
+
+class SummaryAPIConfig(BaseSettings):
+    host: str
+    port: int
+    number_workers: int
+
+
+def get_config() -> SummaryAPIConfig:
+    """
+    Return the configuration for the summary API.
+    The configuration is loaded from a toml file.
+    The path to the toml file can be specified using the environment variable `SUMMARY_API_CONFIG_FILE`.
+    If the environment variable is not set, the default configuration file is used.
+    """
+    config_file = os.environ.get("SUMMARY_API_CONFIG_FILE", None)
+    if config_file is None:
+        config_file = DEFAULT_CONFIG_FILE
+    with open(config_file, "rb") as f:
+        return SummaryAPIConfig(**tomllib.load(f))

bertrend-0.4.1/bertrend/services/summary_server/main.py

@@ -0,0 +1,12 @@
+#  Copyright (c) 2024-2026, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+
+from fastapi import FastAPI
+
+from bertrend.services.summary_server.routers import info, summarize
+
+app = FastAPI(title="BERTrend Summarization Service")
+app.include_router(info.router, tags=["Info"])
+app.include_router(summarize.router, tags=["Summarization"])

bertrend-0.4.1/bertrend/services/summary_server/models.py

@@ -0,0 +1,30 @@
+#  Copyright (c) 2024-2026, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+
+from pydantic import BaseModel
+
+
+class SummarizeRequest(BaseModel):
+    text: str | list[str]
+    summarizer_type: str = "llm"
+    language: str = "fr"
+    max_sentences: int = 3
+    max_words: int | None = None
+    max_length_ratio: float = 0.2
+
+
+class SummarizeResponse(BaseModel):
+    summaries: list[str]
+    summarizer_type: str
+    language: str
+    processing_time_ms: float
+
+
+class SummarizerInfo(BaseModel):
+    name: str
+    description: str
+    requires_api_key: bool
+    requires_gpu: bool
+    loaded: bool

bertrend-0.4.1/bertrend/services/summary_server/routers/info.py

@@ -0,0 +1,33 @@
+#  Copyright (c) 2024-2026, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+
+from fastapi import APIRouter
+
+from bertrend.services.summary_server.models import SummarizerInfo
+from bertrend.services.summary_server.routers.summarize import (
+    SUMMARIZER_REGISTRY,
+    _summarizer_cache,
+)
+
+router = APIRouter()
+
+
+@router.get("/health")
+def health():
+    return {"status": "ok"}
+
+
+@router.get("/summarizers", response_model=list[SummarizerInfo])
+def list_summarizers():
+    return [
+        SummarizerInfo(
+            name=name,
+            description=meta["description"],
+            requires_api_key=meta["requires_api_key"],
+            requires_gpu=meta["requires_gpu"],
+            loaded=name in _summarizer_cache,
+        )
+        for name, meta in SUMMARIZER_REGISTRY.items()
+    ]