bertrend 0.4.2__tar.gz → 0.4.4__tar.gz

{bertrend-0.4.2 → bertrend-0.4.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bertrend
-Version: 0.4.2
+Version: 0.4.4
 Summary: BERTrend analyses topic evolution over time using state-of-the-art transformer models: topic modelling and LLM
 License: LICENSE.md
 License-File: AUTHORS.txt
@@ -12,75 +12,69 @@ Maintainer: Jérôme Picault
 Maintainer-email: jerome.picault@rte-france.com
 Requires-Python: >=3.12,<4.0
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
-Requires-Dist: APScheduler (>=3.11.0)
-Requires-Dist: PyJWT (>=2.10.1)
-Requires-Dist: Pyqtree (==1.0.0)
-Requires-Dist: SQLAlchemy (>=2.0.40)
+Requires-Dist: APScheduler (>=3.11.2)
+Requires-Dist: PyJWT (>=2.11.0)
+Requires-Dist: SQLAlchemy (>=2.0.46)
 Requires-Dist: aio-pika (>=9.5.5)
-Requires-Dist: arxiv (>=2.2.0)
-Requires-Dist: bertopic (==0.16.2)
-Requires-Dist: black (>=24.10.0)
-Requires-Dist: coverage (>=7.6.9)
-Requires-Dist: cron-descriptor (>=1.4.5)
-Requires-Dist: dask (==2024.12.0)
-Requires-Dist: datamapplot (==0.3.0)
-Requires-Dist: dateparser (>=1.2.0)
+Requires-Dist: arxiv (>=2.4.0)
+Requires-Dist: bertopic (>=0.16.2)
+Requires-Dist: coverage (>=7.13.2)
+Requires-Dist: cron-descriptor (>=2.0.6)
+Requires-Dist: datamapplot (>=0.6.4)
+Requires-Dist: dateparser (>=1.2.2)
 Requires-Dist: ddgs (>=9.10.0)
-Requires-Dist: dill (>=0.4.0)
-Requires-Dist: fastapi (>=0.120.0)
-Requires-Dist: feedparser (>=6.0.11)
+Requires-Dist: dill (>=0.4.1)
+Requires-Dist: fastapi (>=0.128.0)
+Requires-Dist: feedparser (>=6.0.12)
 Requires-Dist: folium (>=0.20.0)
-Requires-Dist: gensim (>=4.3.3)
+Requires-Dist: gensim (>=4.4.0)
 Requires-Dist: geopy (>=2.4.1)
-Requires-Dist: google-api-python-client (>=2.148.0)
-Requires-Dist: google-auth-oauthlib (>=1.2.1)
+Requires-Dist: google-api-python-client (>=2.189.0)
+Requires-Dist: google-auth-oauthlib (>=1.2.4)
 Requires-Dist: googlenewsdecoder (>=0.1.7)
-Requires-Dist: goose3 (>=3.1.19)
-Requires-Dist: hdbscan (>=0.8.40)
+Requires-Dist: goose3 (>=3.1.21)
+Requires-Dist: hdbscan (>=0.8.41)
 Requires-Dist: ipython (>=8.28.0)
-Requires-Dist: joblib (>=1.4.2)
+Requires-Dist: joblib (>=1.5.3)
 Requires-Dist: jsonlines (>=4.0.0)
-Requires-Dist: langchain-core (>=0.3.59)
+Requires-Dist: langchain-core (>=1.2.9)
 Requires-Dist: langdetect (>=1.0.9)
 Requires-Dist: loguru (>=0.7.3)
-Requires-Dist: lxml-html-clean (>=0.2.2)
-Requires-Dist: markdown (>=3.7)
+Requires-Dist: lxml-html-clean (>=0.4.3)
+Requires-Dist: markdown (>=3.10.1)
 Requires-Dist: newscatcher-sdk (>=1.4.0)
 Requires-Dist: newspaper4k (>=0.9.3.1)
-Requires-Dist: nltk (>=3.9.1)
-Requires-Dist: openai (>=1.77.0)
-Requires-Dist: openai-agents[litellm] (>=0.4.0)
+Requires-Dist: nicegui[highcharts] (>=3.8.0)
+Requires-Dist: nltk (>=3.9.2)
+Requires-Dist: openai (>=2.18.0)
+Requires-Dist: openai-agents[litellm] (>=0.8.3)
 Requires-Dist: openpyxl (>=3.1.5)
-Requires-Dist: opentelemetry-exporter-otlp-proto-grpc (==1.25.0)
-Requires-Dist: pandas (>=2.2.2)
+Requires-Dist: pandas (<3.0.0)
 Requires-Dist: pika (>=1.3.2)
-Requires-Dist: plotly (>=5.24.1)
-Requires-Dist: plotly-resampler (>=0.10.0)
-Requires-Dist: psutil (>=7.2.1)
-Requires-Dist: pyarrow (>=21.0.0)
+Requires-Dist: plotly (>=6.5.2)
+Requires-Dist: plotly-resampler (>=0.11.0)
+Requires-Dist: psutil (>=7.2.2)
 Requires-Dist: pygooglenews (>=0.1.3)
-Requires-Dist: pylabeladjust (==0.1.13)
-Requires-Dist: pytest (>=8.4.0)
+Requires-Dist: pytest
 Requires-Dist: pytest-asyncio
-Requires-Dist: python-dotenv (>=1.1.1)
+Requires-Dist: python-dotenv (>=1.2.1)
 Requires-Dist: requests (>=2.32.3)
-Requires-Dist: ruff (>=0.14.14)
-Requires-Dist: scikit-learn (>=1.5.2)
+Requires-Dist: ruff (>=0.15.0)
+Requires-Dist: scikit-learn (>=1.8.0)
 Requires-Dist: seaborn (>=0.13.2)
-Requires-Dist: sentence-transformers (==3.0.1)
-Requires-Dist: sentencepiece (>=0.2.0)
-Requires-Dist: starlette (>=0.46.2)
-Requires-Dist: streamlit (>=1.51.0)
+Requires-Dist: sentence-transformers (>=5.2.2)
+Requires-Dist: starlette (>=0.52.1)
+Requires-Dist: streamlit (>=1.54.0)
 Requires-Dist: streamlit-code-editor (>=0.1.22)
 Requires-Dist: streamlit-folium (>=0.26.1)
 Requires-Dist: supervisor (>=4.3.0)
 Requires-Dist: thefuzz (>=0.22.1)
-Requires-Dist: tiktoken (>=0.8.0)
-Requires-Dist: tldextract (>=2.0.1)
-Requires-Dist: torch (>=2.7.0)
-Requires-Dist: tqdm (>=4.67.1)
-Requires-Dist: typer (>=0.16.0)
-Requires-Dist: umap-learn (>=0.5.7)
+Requires-Dist: tiktoken (>=0.12.0)
+Requires-Dist: tldextract (>=5.3.1)
+Requires-Dist: torch (>=2.10.0)
+Requires-Dist: tqdm (>=4.67.3)
+Requires-Dist: typer (>=0.21.1)
+Requires-Dist: umap-learn (>=0.5.11)
 Project-URL: Bug Tracker, https://github.com/rte-france/BERTrend/issues
 Project-URL: Documentation, https://github.com/rte-france/BERTrend
 Project-URL: Homepage, https://github.com/rte-france/BERTrend

{bertrend-0.4.2 → bertrend-0.4.4}/bertrend/bertrend_apps/services/queue_monitor/app.py

@@ -25,6 +25,17 @@ import streamlit as st
 from bertrend.bertrend_apps.services.queue_management.rabbitmq_config import (
     RabbitMQConfig,
 )
+from bertrend.demos.demos_utils.icons import (
+    DASHBOARD_ICON,
+    ERROR_ICON,
+    ERROR_OUTLINE_ICON,
+    INFO_ICON,
+    INVENTORY_ICON,
+    REFRESH_ICON,
+    REPLY_ICON,
+    SEARCH_ICON,
+    SETTINGS_ICON,
+)
 
 
 # ---------- Helpers: Management API client ----------
@@ -65,6 +76,17 @@ class RabbitMQAdminClient:
 # ---------- UI Logic & formatters ----------
 
 
+def queue_icon(queue_name: str) -> str:
+    """Returns an icon based on queue name"""
+    return (
+        ERROR_OUTLINE_ICON
+        if "failed" in queue_name
+        else REPLY_ICON
+        if "response" in queue_name
+        else INVENTORY_ICON
+    )
+
+
 def _decode_message_payload(msg: Dict[str, Any]) -> Tuple[str, Any]:
     """Returns (format_type, decoded_object)"""
     body_text = msg.get("payload")
@@ -190,7 +212,7 @@ cfg = RabbitMQConfig()
 admin_url_default = f"http://{cfg.host}:15672"
 
 with st.sidebar:
-    st.title("⚙️ Settings")
+    st.title(f"{SETTINGS_ICON} Settings")
 
     with st.expander("Connection Details", expanded=False):
         mgmt_url = st.text_input(
@@ -220,7 +242,7 @@ with st.sidebar:
     refresh_rate = c2.select_slider(
         "Refresh (s)", options=[5, 10, 30, 60, 120], value=30
     )
-    force_refresh = c3.button("🔄", help="Force refresh the queue data")
+    force_refresh = c3.button(REFRESH_ICON, help="Force refresh the queue data")
 
     if auto_refresh:
         st.caption(f"Refreshing every {refresh_rate}s...")
@@ -261,11 +283,11 @@ except Exception as e:
 
 st.title("📬 BERTrend Queue Monitor")
 if error_msg:
-    st.error(f"Connection Error: {error_msg}")
+    st.error(f"Connection Error: {error_msg}", icon=ERROR_ICON)
     st.stop()
 
 # 1. Global Overview Row
-st.markdown("### 📊 System Overview")
+st.markdown(f"### {DASHBOARD_ICON} System Overview")
 kpi1, kpi2, kpi3, kpi4 = st.columns(4)
 kpi1.metric(
     "Total Backlog", global_stats["total"], help="Total messages in all selected queues"
@@ -283,9 +305,9 @@ st.divider()
 
 # 2. Queue Tabs
 if not selected_queues:
-    st.info("Select queues in the sidebar to begin monitoring.")
+    st.info("Select queues in the sidebar to begin monitoring.", icon=INFO_ICON)
 else:
-    tabs = st.tabs([f"📦 {q}" for q in selected_queues])
+    tabs = st.tabs([f"{queue_icon(q)} {q}" for q in selected_queues])
 
     for tab, qname in zip(tabs, selected_queues):
         info = queue_data.get(qname, {})
@@ -340,10 +362,10 @@ else:
                 )
                 msgs = client.peek_messages(qname, count=fetch_count)
             except Exception as e:
-                st.error(f"Failed to peek: {e}")
+                st.error(f"Failed to peek: {e}", icon=ERROR_ICON)
 
             if not msgs:
-                st.info("Queue is empty or messages are not available.", icon="ℹ️")
+                st.info("Queue is empty or messages are not available.", icon=INFO_ICON)
             else:
                 # Decode all first for filtering/sorting/collecting filter values
                 decoded_msgs = []
@@ -477,7 +499,7 @@ else:
                     )
 
                 # Filtering inputs
-                with st.expander("🔍 Filters", expanded=False):
+                with st.expander("Filters", expanded=True, icon=SEARCH_ICON):
                     fc1, fc2, fc3 = st.columns(3)
 
                     # Options for selectboxes (sorted for better UX)
@@ -538,6 +560,8 @@ else:
                         info_tags.append(d["endpoint"])
                     if d["user"]:
                         info_tags.append(f"User: {d['user']}")
+                    if d["model_id"]:
+                        info_tags.append(f"Model: {d['model_id']}")
 
                     info_str = f" | {' | '.join(info_tags)}" if info_tags else ""
                     label = f"#{idx + 1} | {fmt} | CID: {corr_id}{info_str} | {msg_size} bytes"

{bertrend-0.4.2 → bertrend-0.4.4}/bertrend/demos/demos_utils/icons.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  Copyright (c) 2024-2026, RTE (https://www.rte-france.com)
 #  See AUTHORS.txt
 #  SPDX-License-Identifier: MPL-2.0
 #  This file is part of BERTrend.
@@ -40,6 +40,12 @@ PERSON_REMOVE_ICON = ":material/person_remove:"
 POPULARITY_ICON = ":material/thumbs_up_down:"
 SIGNAL_EVOLUTION_ICON = ":material/signal_cellular_alt:"
 CYCLE_ICON = ":material/cycle:"
+SEARCH_ICON = ":material/search:"
+REFRESH_ICON = ":material/refresh:"
+DASHBOARD_ICON = ":material/dashboard:"
+INVENTORY_ICON = ":material/inventory_2:"
+REPLY_ICON = ":material/reply:"
+ERROR_OUTLINE_ICON = ":material/error_outline:"
 
 JSON_ICON = "🧾"
 PARQUET_ICON = "📦️"

bertrend-0.4.4/bertrend/services/scheduling/dashboard_scheduling.py

@@ -0,0 +1,280 @@
+#  Copyright (c) 2024-2026, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+import hashlib
+from datetime import datetime
+
+import requests
+from nicegui import run, ui
+
+
+def _clamp(x, lo=0, hi=255):
+    return max(lo, min(hi, x))
+
+
+def _adjust_color(hex_color, factor):
+    """
+    Adjust color brightness.
+    factor > 1 -> lighten, factor < 1 -> darken.
+    """
+    hex_color = hex_color.lstrip("#")
+    r = int(hex_color[0:2], 16)
+    g = int(hex_color[2:4], 16)
+    b = int(hex_color[4:6], 16)
+
+    r = _clamp(int(r * factor))
+    g = _clamp(int(g * factor))
+    b = _clamp(int(b * factor))
+
+    return "#{:02x}{:02x}{:02x}".format(r, g, b)
+
+
+def get_color(user, model_id, endpoint):
+    """
+    Generates a stable hex color from (user, model_id).
+    - Lighter for 'scrape' endpoints
+    - Darker for 'report' endpoints
+    """
+    seed = f"{user}-{model_id}".encode("utf-8")
+    hash_hex = hashlib.md5(seed).hexdigest()
+    base_color = f"#{hash_hex[:6]}"
+
+    ep = endpoint.lower()
+    if "scrape" in ep:
+        # lighten ~30%
+        return _adjust_color(base_color, 1.3)
+    if "report" in ep:
+        # darken ~30%
+        return _adjust_color(base_color, 0.7)
+
+    return base_color
+
+
+def parse_jobs(data):
+    parsed = []
+    if not isinstance(data, list):
+        return []
+    for item in data:
+        kwargs = item.get("kwargs", {}) or {}
+        url = kwargs.get("url", "")
+        endpoint = url.split("/")[-1] if url else "N/A"
+        json_data = kwargs.get("json_data", {}) or {}
+        user = str(json_data.get("user", "system"))
+        model_id = json_data.get("model_id", "N/A")
+        parsed.append(
+            {
+                "user": user,
+                "model_id": model_id,
+                "endpoint": endpoint,
+                "time": item.get("next_run_time", ""),
+                "job_id": item.get("job_id", "N/A"),
+                "color": get_color(user, model_id, endpoint),
+            }
+        )
+    return sorted(parsed, key=lambda x: x["time"])
+
+
+class JobDashboard:
+    def __init__(self):
+        self.api_url = "http://dsia.rte-france.com:8882/jobs"
+        self.all_jobs = []
+        self.filtered_jobs = []
+
+    async def fetch_data(self):
+        try:
+            response = await run.io_bound(requests.get, self.api_url, timeout=10)
+            if response.status_code == 200:
+                self.all_jobs = parse_jobs(response.json())
+                self.filtered_jobs = self.all_jobs[:]
+                ui.notify(f"Fetched {len(self.all_jobs)} jobs", type="positive")
+            else:
+                ui.notify(f"Error: {response.status_code}", type="negative")
+        except Exception as e:
+            ui.notify(f"Connection Failed: {e}", type="negative")
+
+
+@ui.page("/")
+async def main_page():
+    # Load Highcharts Module
+    ui.add_head_html(
+        '<script src="https://code.highcharts.com/modules/timeline.js"></script>'
+    )
+    ui.colors(primary="#0052a1")
+    dashboard = JobDashboard()
+
+    # --- LOGIC ---
+    def update_display():
+        f_users = user_filter.value
+        f_models = model_filter.value
+        f_urls = url_filter.value
+
+        dashboard.filtered_jobs = [
+            j
+            for j in dashboard.all_jobs
+            if (not f_users or j["user"] in f_users)
+            and (not f_models or j["model_id"] in f_models)
+            and (not f_urls or j["endpoint"] in f_urls)
+        ]
+        render_content()
+
+    def reset_filters():
+        user_filter.value = []
+        model_filter.value = []
+        url_filter.value = []
+        update_display()
+
+    def render_content():
+        # Clear both containers
+        chart_container.clear()
+        timeline_container.clear()
+
+        if not dashboard.filtered_jobs:
+            with chart_container:
+                ui.label("No jobs match selected filters.").classes(
+                    "mx-auto py-10 text-slate-400"
+                )
+            return
+
+        # 1. Update Chart Dashboard
+        with chart_container:
+            chart_options = {
+                "chart": {"type": "timeline", "height": 350, "zoomType": "x"},
+                "title": {
+                    "text": "Job Schedule Timeline",
+                    "style": {"fontSize": "14px"},
+                },
+                "xAxis": {"type": "datetime"},
+                "series": [
+                    {
+                        "data": [
+                            {
+                                "x": int(
+                                    datetime.fromisoformat(j["time"]).timestamp() * 1000
+                                ),
+                                "name": f"/{j['endpoint']}",
+                                "label": f"{j['user']} | {j['model_id']}",
+                                "color": j["color"],
+                            }
+                            for j in dashboard.filtered_jobs
+                            if j["time"]
+                        ]
+                    }
+                ],
+            }
+            ui.highchart(chart_options, extras=["timeline"]).classes("w-full")
+
+        # 2. Update Timeline List Dashboard
+        with timeline_container:
+            ui.label("Detailed Job Queue").classes(
+                "text-sm font-bold text-slate-500 mb-4 uppercase"
+            )
+            # --- Generate and inject CSS for our custom hex colors ---
+            unique_colors = set(job["color"] for job in dashboard.filtered_jobs)
+            css_rules = []
+            for hex_color in unique_colors:
+                # Create a valid CSS identifier from the hex (e.g., #ff0000 -> c_ff0000)
+                class_name = f"c_{hex_color.lstrip('#')}"
+                css_rules.append(
+                    f".text-{class_name} {{ color: {hex_color} !important; }}"
+                )
+                css_rules.append(
+                    f".bg-{class_name} {{ background-color: {hex_color} !important; }}"
+                )
+
+            if css_rules:
+                ui.add_css("\n".join(css_rules))
+            # --------------------------------------------------------------
+            with ui.timeline(side="right"):
+                for job in dashboard.filtered_jobs:
+                    custom_color_name = f"c_{job['color'].lstrip('#')}"
+                    ui.timeline_entry(
+                        f"Model: {job['model_id']}",
+                        title=f"/{job['endpoint']} ({job['user']})",
+                        subtitle=job["time"],
+                        color=custom_color_name,
+                    )
+
+    async def refresh_all():
+        await dashboard.fetch_data()
+        # Refresh options based on new data
+        user_filter.set_options(
+            sorted(list(set(j["user"] for j in dashboard.all_jobs)))
+        )
+        model_filter.set_options(
+            sorted(list(set(j["model_id"] for j in dashboard.all_jobs)))
+        )
+        url_filter.set_options(
+            sorted(list(set(j["endpoint"] for j in dashboard.all_jobs)))
+        )
+        update_display()
+
+    # --- UI LAYOUT ---
+    with ui.header().classes("items-center justify-between px-4 py-2"):
+        with ui.row().classes("items-center"):
+            ui.icon("analytics", size="md")
+            ui.label("BERTrend Scheduler").classes("text-lg font-bold")
+
+        # Endpoint Config in Header
+        with ui.row().classes("items-center bg-white/10 p-1 rounded-md"):
+            ui.input(value=dashboard.api_url).bind_value(dashboard, "api_url").props(
+                "dark dense borderless"
+            ).classes("w-64 px-2 text-sm")
+            ui.button(icon="refresh", on_click=refresh_all).props(
+                "flat color=white dense"
+            )
+
+    # Filter Bar
+    with ui.row().classes("w-full items-center p-2 bg-slate-50 border-b gap-3"):
+        ui.label("FILTERS:").classes("text-xs font-bold text-slate-500 ml-2")
+
+        url_filter = (
+            ui.select(
+                [],
+                label="URL Type",
+                multiple=True,
+                with_input=True,
+                on_change=update_display,
+            )
+            .props("dense outlined")
+            .classes("w-48 text-xs bg-white")
+        )
+
+        user_filter = (
+            ui.select(
+                [],
+                label="User",
+                multiple=True,
+                with_input=True,
+                on_change=update_display,
+            )
+            .props("dense outlined")
+            .classes("w-40 text-xs bg-white")
+        )
+
+        model_filter = (
+            ui.select(
+                [],
+                label="Model ID",
+                multiple=True,
+                with_input=True,
+                on_change=update_display,
+            )
+            .props("dense outlined")
+            .classes("w-40 text-xs bg-white")
+        )
+
+        ui.button("Reset", on_click=reset_filters, icon="filter_alt_off").props(
+            "outline dense"
+        ).classes("text-xs")
+
+    # The two Dashboard Containers
+    chart_container = ui.column().classes("w-full p-4")
+    ui.separator()
+    timeline_container = ui.column().classes("w-full p-4")
+
+    # Initial Load
+    await refresh_all()
+
+
+ui.run(title="BERTrend Job Viewer", port=8885)